From a3e9b08c4a01929bbc670da33e4f37df7a5eee6f Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 25 Oct 2022 20:25:43 +0800
Subject: [PATCH 01/33] executor: support Intersection of IndexMerge

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 308 ++++++++++++++++++++++++++-------
 1 file changed, 241 insertions(+), 67 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index bc29199a2c2b7..a32d80dd40447 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -94,8 +94,8 @@ type IndexMergeReaderExecutor struct {
 	workerStarted bool
 	keyRanges     [][]kv.KeyRange
 
-	resultCh   chan *lookupTableTask
-	resultCurr *lookupTableTask
+	resultCh   chan *indexMergeTableTask
+	resultCurr *indexMergeTableTask
 	feedbacks  []*statistics.QueryFeedback
 
 	// memTracker is used to track the memory usage of this executor.
@@ -118,6 +118,15 @@ type IndexMergeReaderExecutor struct {
 	isCorColInPartialFilters []bool
 	isCorColInTableFilter    bool
 	isCorColInPartialAccess  []bool
+
+	isIntersection bool
+}
+
+type indexMergeTableTask struct {
+	lookupTableTask
+
+	workerID  int
+	parTblIdx int
 }
 
 // Table implements the dataSourceExecutor interface.
@@ -150,7 +159,7 @@ func (e *IndexMergeReaderExecutor) Open(ctx context.Context) (err error) {
 		}
 	}
 	e.finished = make(chan struct{})
-	e.resultCh = make(chan *lookupTableTask, atomic.LoadInt32(&LookupTableTaskChannelSize))
+	e.resultCh = make(chan *indexMergeTableTask, atomic.LoadInt32(&LookupTableTaskChannelSize))
 	e.memTracker = memory.NewTracker(e.id, -1)
 	e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker)
 	return nil
@@ -209,8 +218,8 @@ func (e *IndexMergeReaderExecutor) buildKeyRangesForTable(tbl table.Table) (rang
 
 func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
 	exitCh := make(chan struct{})
-	workCh := make(chan *lookupTableTask, 1)
-	fetchCh := make(chan *lookupTableTask, len(e.keyRanges))
+	workCh := make(chan *indexMergeTableTask, 1)
+	fetchCh := make(chan *indexMergeTableTask, len(e.keyRanges))
 
 	e.startIndexMergeProcessWorker(ctx, workCh, fetchCh)
 
@@ -237,12 +246,12 @@ func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
 	return nil
 }
 
-func (e *IndexMergeReaderExecutor) waitPartialWorkersAndCloseFetchChan(fetchCh chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) waitPartialWorkersAndCloseFetchChan(fetchCh chan *indexMergeTableTask) {
 	e.idxWorkerWg.Wait()
 	close(fetchCh)
 }
 
-func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Context, workCh chan<- *lookupTableTask, fetch <-chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Context, workCh chan<- *indexMergeTableTask, fetch <-chan *indexMergeTableTask) {
 	idxMergeProcessWorker := &indexMergeProcessWorker{
 		indexMerge: e,
 		stats:      e.stats,
@@ -252,7 +261,10 @@ func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Cont
 		defer trace.StartRegion(ctx, "IndexMergeProcessWorker").End()
 		util.WithRecovery(
 			func() {
-				idxMergeProcessWorker.fetchLoop(ctx, fetch, workCh, e.resultCh, e.finished)
+				if e.isIntersection {
+					idxMergeProcessWorker.fetchLoopIntersection(ctx, fetch, workCh, e.resultCh, e.finished)
+				}
+				idxMergeProcessWorker.fetchLoopUnion(ctx, fetch, workCh, e.resultCh, e.finished)
 			},
 			idxMergeProcessWorker.handleLoopFetcherPanic(ctx, e.resultCh),
 		)
@@ -260,7 +272,7 @@ func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Cont
 	}()
 }
 
-func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, workID int) error {
+func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, workID int) error {
 	if e.runtimeStats != nil {
 		collExec := true
 		e.dagPBs[workID].CollectExecutionSummaries = &collExec
@@ -291,13 +303,14 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
 					batchSize:    e.maxChunkSize,
 					maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
 					maxChunkSize: e.maxChunkSize,
+					workerID:     workID,
 				}
 
 				if e.isCorColInPartialFilters[workID] {
 					// We got correlated column, so need to refresh Selection operator.
 					var err error
 					if e.dagPBs[workID].Executors, err = constructDistExec(e.ctx, e.partialPlans[workID]); err != nil {
-						worker.syncErr(e.resultCh, err)
+						syncErr(e.resultCh, err)
 						return
 					}
 				}
@@ -331,12 +344,12 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
 					})
 					kvReq, err := builder.SetKeyRanges(keyRange).Build()
 					if err != nil {
-						worker.syncErr(e.resultCh, err)
+						syncErr(e.resultCh, err)
 						return
 					}
 					result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, e.handleCols.GetFieldsTypes(), e.feedbacks[workID], getPhysicalPlanIDs(e.partialPlans[workID]), e.getPartitalPlanID(workID))
 					if err != nil {
-						worker.syncErr(e.resultCh, err)
+						syncErr(e.resultCh, err)
 						return
 					}
 					worker.batchSize = e.maxChunkSize
@@ -349,7 +362,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
 
 					// fetch all data from this partition
 					ctx1, cancel := context.WithCancel(ctx)
-					_, fetchErr := worker.fetchHandles(ctx1, result, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols)
+					_, fetchErr := worker.fetchHandles(ctx1, result, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols, parTblIdx)
 					if fetchErr != nil { // this error is synced in fetchHandles(), don't sync it again
 						e.feedbacks[workID].Invalidate()
 					}
@@ -370,7 +383,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
 	return nil
 }
 
-func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, workID int) error {
+func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, workID int) error {
 	ts := e.partialPlans[workID][0].(*plannercore.PhysicalTableScan)
 
 	tbls := make([]table.Table, 0, 1)
@@ -408,17 +421,18 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
 					maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
 					maxChunkSize: e.maxChunkSize,
 					tableReader:  partialTableReader,
+					workerID:     workID,
 				}
 
 				if e.isCorColInPartialFilters[workID] {
 					if e.dagPBs[workID].Executors, err = constructDistExec(e.ctx, e.partialPlans[workID]); err != nil {
-						worker.syncErr(e.resultCh, err)
+						syncErr(e.resultCh, err)
 						return
 					}
 					partialTableReader.dagPB = e.dagPBs[workID]
 				}
 
-				for _, tbl := range tbls {
+				for parTblIdx, tbl := range tbls {
 					// check if this executor is closed
 					select {
 					case <-e.finished:
@@ -430,7 +444,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
 					partialTableReader.table = tbl
 					if err = partialTableReader.Open(ctx); err != nil {
 						logutil.Logger(ctx).Error("open Select result failed:", zap.Error(err))
-						worker.syncErr(e.resultCh, err)
+						syncErr(e.resultCh, err)
 						break
 					}
 					worker.batchSize = e.maxChunkSize
@@ -443,7 +457,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
 
 					// fetch all handles from this table
 					ctx1, cancel := context.WithCancel(ctx)
-					_, fetchErr := worker.fetchHandles(ctx1, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols)
+					_, fetchErr := worker.fetchHandles(ctx1, exitCh, fetchCh, e.resultCh, e.finished, e.handleCols, parTblIdx)
 					if fetchErr != nil { // this error is synced in fetchHandles, so don't sync it again
 						e.feedbacks[workID].Invalidate()
 					}
@@ -496,18 +510,11 @@ type partialTableWorker struct {
 	maxChunkSize int
 	tableReader  Executor
 	partition    table.PhysicalTable // it indicates if this worker is accessing a particular partition table
+	workerID     int
 }
 
-func (w *partialTableWorker) syncErr(resultCh chan<- *lookupTableTask, err error) {
-	doneCh := make(chan error, 1)
-	doneCh <- err
-	resultCh <- &lookupTableTask{
-		doneCh: doneCh,
-	}
-}
-
-func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, resultCh chan<- *lookupTableTask,
-	finished <-chan struct{}, handleCols plannercore.HandleCols) (count int64, err error) {
+func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask,
+	finished <-chan struct{}, handleCols plannercore.HandleCols, parTblIdx int) (count int64, err error) {
 	chk := chunk.NewChunkWithCapacity(retTypes(w.tableReader), w.maxChunkSize)
 	var basic *execdetails.BasicRuntimeStats
 	if be := w.tableReader.base(); be != nil && be.runtimeStats != nil {
@@ -517,14 +524,14 @@ func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan str
 		start := time.Now()
 		handles, retChunk, err := w.extractTaskHandles(ctx, chk, handleCols)
 		if err != nil {
-			w.syncErr(resultCh, err)
+			syncErr(resultCh, err)
 			return count, err
 		}
 		if len(handles) == 0 {
 			return count, nil
 		}
 		count += int64(len(handles))
-		task := w.buildTableTask(handles, retChunk)
+		task := w.buildTableTask(handles, retChunk, parTblIdx)
 		if w.stats != nil {
 			atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start)))
 		}
@@ -570,19 +577,23 @@ func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.
 	return handles, retChk, nil
 }
 
-func (w *partialTableWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk) *lookupTableTask {
-	task := &lookupTableTask{
-		handles: handles,
-		idxRows: retChk,
+func (w *partialTableWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk, parTblIdx int) *indexMergeTableTask {
+	task := &indexMergeTableTask{
+		lookupTableTask: lookupTableTask{
+			handles: handles,
+			idxRows: retChk,
 
-		partitionTable: w.partition,
+			partitionTable: w.partition,
+		},
+		workerID:  w.workerID,
+		parTblIdx: parTblIdx,
 	}
 
 	task.doneCh = make(chan error, 1)
 	return task
 }
 
-func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Context, workCh <-chan *lookupTableTask) {
+func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Context, workCh <-chan *indexMergeTableTask) {
 	lookupConcurrencyLimit := e.ctx.GetSessionVars().IndexLookupConcurrency()
 	e.tblWorkerWg.Add(lookupConcurrencyLimit)
 	for i := 0; i < lookupConcurrencyLimit; i++ {
@@ -597,7 +608,7 @@ func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Co
 		ctx1, cancel := context.WithCancel(ctx)
 		go func() {
 			defer trace.StartRegion(ctx, "IndexMergeTableScanWorker").End()
-			var task *lookupTableTask
+			var task *indexMergeTableTask
 			util.WithRecovery(
 				func() { task = worker.pickAndExecTask(ctx1) },
 				worker.handlePickAndExecTaskPanic(ctx1, task),
@@ -666,7 +677,7 @@ func (e *IndexMergeReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) e
 	}
 }
 
-func (e *IndexMergeReaderExecutor) getResultTask() (*lookupTableTask, error) {
+func (e *IndexMergeReaderExecutor) getResultTask() (*indexMergeTableTask, error) {
 	if e.resultCurr != nil && e.resultCurr.cursor < len(e.resultCurr.rows) {
 		return e.resultCurr, nil
 	}
@@ -686,7 +697,7 @@ func (e *IndexMergeReaderExecutor) getResultTask() (*lookupTableTask, error) {
 	return e.resultCurr, nil
 }
 
-func (e *IndexMergeReaderExecutor) handleHandlesFetcherPanic(ctx context.Context, resultCh chan<- *lookupTableTask, worker string) func(r interface{}) {
+func (e *IndexMergeReaderExecutor) handleHandlesFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask, worker string) func(r interface{}) {
 	return func(r interface{}) {
 		if r == nil {
 			return
@@ -696,8 +707,10 @@ func (e *IndexMergeReaderExecutor) handleHandlesFetcherPanic(ctx context.Context
 		logutil.Logger(ctx).Error(err4Panic.Error())
 		doneCh := make(chan error, 1)
 		doneCh <- err4Panic
-		resultCh <- &lookupTableTask{
-			doneCh: doneCh,
+		resultCh <- &indexMergeTableTask{
+			lookupTableTask: lookupTableTask{
+				doneCh: doneCh,
+			},
 		}
 	}
 }
@@ -722,8 +735,8 @@ type indexMergeProcessWorker struct {
 	stats      *IndexMergeRuntimeStat
 }
 
-func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan *lookupTableTask,
-	workCh chan<- *lookupTableTask, resultCh chan<- *lookupTableTask, finished <-chan struct{}) {
+func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
+	workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
 	defer func() {
 		close(workCh)
 		close(resultCh)
@@ -755,15 +768,123 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan
 		if len(fhs) == 0 {
 			continue
 		}
-		task := &lookupTableTask{
-			handles: fhs,
-			doneCh:  make(chan error, 1),
+		task := &indexMergeTableTask{
+			lookupTableTask: lookupTableTask{
+				handles: fhs,
+				doneCh:  make(chan error, 1),
 
-			partitionTable: task.partitionTable,
+				partitionTable: task.partitionTable,
+			},
+		}
+		if w.stats != nil {
+			w.stats.IndexMergeProcess += time.Since(start)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-finished:
+			return
+		case workCh <- task:
+			resultCh <- task
+		}
+	}
+}
+
+func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
+	workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
+	defer func() {
+		close(workCh)
+		close(resultCh)
+	}()
+
+	// key: handle, value: partIdx for this handle.
+	partHandleMap := kv.NewHandleMap()
+	// key: partialWorker, value: handles for this partialWorker.
+	workerHandleMap := make(map[int]*kv.HandleMap)
+	for i := 0; i < len(w.indexMerge.partialPlans); i++ {
+		workerHandleMap[i] = kv.NewHandleMap()
+	}
+	// Distinct handles for each partialWorker.
+	for task := range fetchCh {
+		start := time.Now()
+		handles := task.handles
+
+		if _, ok := workerHandleMap[task.workerID]; !ok {
+			err := errors.New(fmt.Sprintf("unexpected workerID, got %d, total: %d", task.workerID, len(w.indexMerge.partialPlans)))
+			syncErr(resultCh, err)
+			return
+		}
+		hMap := workerHandleMap[task.workerID]
+
+		for _, h := range handles {
+			if _, ok := hMap.Get(h); !ok {
+				hMap.Set(h, true)
+			}
+			if w.indexMerge.partitionTableMode {
+				if parTblIdx, ok := partHandleMap.Get(h); ok {
+					if parTblIdx.(int) != task.parTblIdx {
+						err := errors.New(fmt.Sprintf("expected parTblIdx: %v, got: %d", parTblIdx, task.parTblIdx))
+						syncErr(resultCh, err)
+						return
+					}
+				} else {
+					partHandleMap.Set(h, task.parTblIdx)
+				}
+			}
 		}
 		if w.stats != nil {
 			w.stats.IndexMergeProcess += time.Since(start)
 		}
+	}
+
+	start := time.Now()
+	intersected := intersectMaps(workerHandleMap)
+	if w.stats != nil {
+		w.stats.IndexMergeProcess += time.Since(start)
+	}
+	if len(intersected) == 0 {
+		// No data to send, just return, resultCh and workCh will be closed in defer func.
+		return
+	}
+	var tasks []*indexMergeTableTask
+	if !w.indexMerge.partitionTableMode {
+		tasks = append(tasks, &indexMergeTableTask{
+			lookupTableTask: lookupTableTask{
+				handles: intersected,
+				doneCh:  make(chan error, 1),
+			},
+		})
+	} else {
+		// key: partIdx, value: intersected handles of this partIdx.
+		intersectedPartHandleMap := make(map[int][]kv.Handle)
+		for _, h := range intersected {
+			var parTblIdx int
+			if parTblIdxI, ok := partHandleMap.Get(h); !ok {
+				err := errors.New("parTblIdx not found")
+				syncErr(resultCh, err)
+				return
+			} else {
+				parTblIdx = parTblIdxI.(int)
+			}
+
+			if _, ok := intersectedPartHandleMap[parTblIdx]; !ok {
+				intersectedPartHandleMap[parTblIdx] = make([]kv.Handle, 0, 8)
+			}
+			intersectedPartHandleMap[parTblIdx] = append(intersectedPartHandleMap[parTblIdx], h)
+		}
+
+		for parTblIdx, handles := range intersectedPartHandleMap {
+			tasks = append(tasks, &indexMergeTableTask{
+				lookupTableTask: lookupTableTask{
+					handles: handles,
+					doneCh:  make(chan error, 1),
+
+					partitionTable: w.indexMerge.prunedPartitions[parTblIdx],
+				},
+			})
+		}
+	}
+	for _, task := range tasks {
 		select {
 		case <-ctx.Done():
 			return
@@ -775,7 +896,50 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan
 	}
 }
 
-func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *lookupTableTask) func(r interface{}) {
+func intersectMaps(distinctHandles map[int]*kv.HandleMap) (res []kv.Handle) {
+	if len(distinctHandles) == 0 {
+		return nil
+	}
+	var handleMaps []*kv.HandleMap
+	var gotEmptyHandleMap bool
+	for _, m := range distinctHandles {
+		if m.Len() == 0 {
+			gotEmptyHandleMap = true
+		}
+		handleMaps = append(handleMaps, m)
+	}
+	if gotEmptyHandleMap {
+		return nil
+	}
+
+	intersected := handleMaps[0]
+	if len(handleMaps) > 1 {
+		for i := 1; i < len(handleMaps); i++ {
+			if intersected.Len() == 0 {
+				break
+			}
+			intersected = intersectTwoMaps(intersected, handleMaps[i])
+		}
+	}
+	intersected.Range(func(h kv.Handle, val interface{}) bool {
+		res = append(res, h)
+		return true
+	})
+	return
+}
+
+func intersectTwoMaps(m1, m2 *kv.HandleMap) *kv.HandleMap {
+	intersected := kv.NewHandleMap()
+	m1.Range(func(h kv.Handle, val interface{}) bool {
+		if _, ok := m2.Get(h); ok {
+			intersected.Set(h, true)
+		}
+		return true
+	})
+	return intersected
+}
+
+func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask) func(r interface{}) {
 	return func(r interface{}) {
 		if r == nil {
 			return
@@ -785,8 +949,10 @@ func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, re
 		logutil.Logger(ctx).Error(err4Panic.Error())
 		doneCh := make(chan error, 1)
 		doneCh <- err4Panic
-		resultCh <- &lookupTableTask{
-			doneCh: doneCh,
+		resultCh <- &indexMergeTableTask{
+			lookupTableTask: lookupTableTask{
+				doneCh: doneCh,
+			},
 		}
 	}
 }
@@ -799,13 +965,16 @@ type partialIndexWorker struct {
 	maxBatchSize int
 	maxChunkSize int
 	partition    table.PhysicalTable // it indicates if this worker is accessing a particular partition table
+	workerID     int
 }
 
-func (w *partialIndexWorker) syncErr(resultCh chan<- *lookupTableTask, err error) {
+func syncErr(resultCh chan<- *indexMergeTableTask, err error) {
 	doneCh := make(chan error, 1)
 	doneCh <- err
-	resultCh <- &lookupTableTask{
-		doneCh: doneCh,
+	resultCh <- &indexMergeTableTask{
+		lookupTableTask: lookupTableTask{
+			doneCh: doneCh,
+		},
 	}
 }
 
@@ -813,10 +982,11 @@ func (w *partialIndexWorker) fetchHandles(
 	ctx context.Context,
 	result distsql.SelectResult,
 	exitCh <-chan struct{},
-	fetchCh chan<- *lookupTableTask,
-	resultCh chan<- *lookupTableTask,
+	fetchCh chan<- *indexMergeTableTask,
+	resultCh chan<- *indexMergeTableTask,
 	finished <-chan struct{},
-	handleCols plannercore.HandleCols) (count int64, err error) {
+	handleCols plannercore.HandleCols,
+	parTblIdx int) (count int64, err error) {
 	chk := chunk.NewChunkWithCapacity(handleCols.GetFieldsTypes(), w.maxChunkSize)
 	var basicStats *execdetails.BasicRuntimeStats
 	if w.stats != nil {
@@ -829,7 +999,7 @@ func (w *partialIndexWorker) fetchHandles(
 		start := time.Now()
 		handles, retChunk, err := w.extractTaskHandles(ctx, chk, result, handleCols)
 		if err != nil {
-			w.syncErr(resultCh, err)
+			syncErr(resultCh, err)
 			return count, err
 		}
 		if len(handles) == 0 {
@@ -839,7 +1009,7 @@ func (w *partialIndexWorker) fetchHandles(
 			return count, nil
 		}
 		count += int64(len(handles))
-		task := w.buildTableTask(handles, retChunk)
+		task := w.buildTableTask(handles, retChunk, parTblIdx)
 		if w.stats != nil {
 			atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start)))
 		}
@@ -885,12 +1055,16 @@ func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.
 	return handles, retChk, nil
 }
 
-func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk) *lookupTableTask {
-	task := &lookupTableTask{
-		handles: handles,
-		idxRows: retChk,
+func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.Chunk, parTblIdx int) *indexMergeTableTask {
+	task := &indexMergeTableTask{
+		lookupTableTask: lookupTableTask{
+			handles: handles,
+			idxRows: retChk,
 
-		partitionTable: w.partition,
+			partitionTable: w.partition,
+		},
+		workerID:  w.workerID,
+		parTblIdx: parTblIdx,
 	}
 
 	task.doneCh = make(chan error, 1)
@@ -899,7 +1073,7 @@ func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.C
 
 type indexMergeTableScanWorker struct {
 	stats          *IndexMergeRuntimeStat
-	workCh         <-chan *lookupTableTask
+	workCh         <-chan *indexMergeTableTask
 	finished       <-chan struct{}
 	indexMergeExec *IndexMergeReaderExecutor
 	tblPlans       []plannercore.PhysicalPlan
@@ -908,7 +1082,7 @@ type indexMergeTableScanWorker struct {
 	memTracker *memory.Tracker
 }
 
-func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task *lookupTableTask) {
+func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task *indexMergeTableTask) {
 	var ok bool
 	for {
 		waitStart := time.Now()
@@ -931,7 +1105,7 @@ func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task *
 	}
 }
 
-func (w *indexMergeTableScanWorker) handlePickAndExecTaskPanic(ctx context.Context, task *lookupTableTask) func(r interface{}) {
+func (w *indexMergeTableScanWorker) handlePickAndExecTaskPanic(ctx context.Context, task *indexMergeTableTask) func(r interface{}) {
 	return func(r interface{}) {
 		if r == nil {
 			return
@@ -943,7 +1117,7 @@ func (w *indexMergeTableScanWorker) handlePickAndExecTaskPanic(ctx context.Conte
 	}
 }
 
-func (w *indexMergeTableScanWorker) executeTask(ctx context.Context, task *lookupTableTask) error {
+func (w *indexMergeTableScanWorker) executeTask(ctx context.Context, task *indexMergeTableTask) error {
 	tbl := w.indexMergeExec.table
 	if w.indexMergeExec.partitionTableMode {
 		tbl = task.partitionTable

From 2c1f53bcedb21099776198f9708f1212557dd883 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 25 Oct 2022 21:04:21 +0800
Subject: [PATCH 02/33] fix fmt

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index a32d80dd40447..8183d3837e932 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -810,7 +810,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		handles := task.handles
 
 		if _, ok := workerHandleMap[task.workerID]; !ok {
-			err := errors.New(fmt.Sprintf("unexpected workerID, got %d, total: %d", task.workerID, len(w.indexMerge.partialPlans)))
+			err := errors.Errorf("unexpected workerID, got %d, total: %d", task.workerID, len(w.indexMerge.partialPlans))
 			syncErr(resultCh, err)
 			return
 		}
@@ -823,7 +823,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			if w.indexMerge.partitionTableMode {
 				if parTblIdx, ok := partHandleMap.Get(h); ok {
 					if parTblIdx.(int) != task.parTblIdx {
-						err := errors.New(fmt.Sprintf("expected parTblIdx: %v, got: %d", parTblIdx, task.parTblIdx))
+						err := errors.Errorf("expected parTblIdx: %v, got: %d", parTblIdx, task.parTblIdx)
 						syncErr(resultCh, err)
 						return
 					}
@@ -858,14 +858,13 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		// key: partIdx, value: intersected handles of this partIdx.
 		intersectedPartHandleMap := make(map[int][]kv.Handle)
 		for _, h := range intersected {
-			var parTblIdx int
-			if parTblIdxI, ok := partHandleMap.Get(h); !ok {
-				err := errors.New("parTblIdx not found")
+			parTblIdxI, ok := partHandleMap.Get(h)
+			if !ok {
+				err := errors.New("handle not found")
 				syncErr(resultCh, err)
 				return
-			} else {
-				parTblIdx = parTblIdxI.(int)
 			}
+			parTblIdx := parTblIdxI.(int)
 
 			if _, ok := intersectedPartHandleMap[parTblIdx]; !ok {
 				intersectedPartHandleMap[parTblIdx] = make([]kv.Handle, 0, 8)
@@ -900,7 +899,7 @@ func intersectMaps(distinctHandles map[int]*kv.HandleMap) (res []kv.Handle) {
 	if len(distinctHandles) == 0 {
 		return nil
 	}
-	var handleMaps []*kv.HandleMap
+	handleMaps := make([]*kv.HandleMap, 0, len(distinctHandles))
 	var gotEmptyHandleMap bool
 	for _, m := range distinctHandles {
 		if m.Len() == 0 {

From 3f6dbb54eda97bbbadb26ab356f71f3c3e7465ad Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Thu, 27 Oct 2022 10:55:24 +0800
Subject: [PATCH 03/33] fix trivial bug

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 8183d3837e932..cbb0c13da5079 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -119,6 +119,7 @@ type IndexMergeReaderExecutor struct {
 	isCorColInTableFilter    bool
 	isCorColInPartialAccess  []bool
 
+	// Whether it's intersection or union.
 	isIntersection bool
 }
 
@@ -263,8 +264,9 @@ func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Cont
 			func() {
 				if e.isIntersection {
 					idxMergeProcessWorker.fetchLoopIntersection(ctx, fetch, workCh, e.resultCh, e.finished)
+				} else {
+					idxMergeProcessWorker.fetchLoopUnion(ctx, fetch, workCh, e.resultCh, e.finished)
 				}
-				idxMergeProcessWorker.fetchLoopUnion(ctx, fetch, workCh, e.resultCh, e.finished)
 			},
 			idxMergeProcessWorker.handleLoopFetcherPanic(ctx, e.resultCh),
 		)
@@ -838,10 +840,13 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	}
 
 	start := time.Now()
+	defer func() {
+		if w.stats != nil {
+			w.stats.IndexMergeProcess += time.Since(start)
+		}
+	}()
+	// Intersect handles among all partialWorkers.
 	intersected := intersectMaps(workerHandleMap)
-	if w.stats != nil {
-		w.stats.IndexMergeProcess += time.Since(start)
-	}
 	if len(intersected) == 0 {
 		// No data to send, just return, resultCh and workCh will be closed in defer func.
 		return
@@ -855,6 +860,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			},
 		})
 	} else {
+		// Need to distinguish which parition table this handle belongs to.
 		// key: partIdx, value: intersected handles of this partIdx.
 		intersectedPartHandleMap := make(map[int][]kv.Handle)
 		for _, h := range intersected {

From fd03889b554ad874d112b31e230aaabdc143cb9c Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Thu, 27 Oct 2022 11:55:35 +0800
Subject: [PATCH 04/33] add unit test

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 45 +---------------------------------
 executor/utils.go              | 45 ++++++++++++++++++++++++++++++++++
 executor/utils_test.go         | 37 ++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 44 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index cbb0c13da5079..3909bec2c89ea 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -846,7 +846,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		}
 	}()
 	// Intersect handles among all partialWorkers.
-	intersected := intersectMaps(workerHandleMap)
+	intersected := IntersectHandleMaps(workerHandleMap)
 	if len(intersected) == 0 {
 		// No data to send, just return, resultCh and workCh will be closed in defer func.
 		return
@@ -901,49 +901,6 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	}
 }
 
-func intersectMaps(distinctHandles map[int]*kv.HandleMap) (res []kv.Handle) {
-	if len(distinctHandles) == 0 {
-		return nil
-	}
-	handleMaps := make([]*kv.HandleMap, 0, len(distinctHandles))
-	var gotEmptyHandleMap bool
-	for _, m := range distinctHandles {
-		if m.Len() == 0 {
-			gotEmptyHandleMap = true
-		}
-		handleMaps = append(handleMaps, m)
-	}
-	if gotEmptyHandleMap {
-		return nil
-	}
-
-	intersected := handleMaps[0]
-	if len(handleMaps) > 1 {
-		for i := 1; i < len(handleMaps); i++ {
-			if intersected.Len() == 0 {
-				break
-			}
-			intersected = intersectTwoMaps(intersected, handleMaps[i])
-		}
-	}
-	intersected.Range(func(h kv.Handle, val interface{}) bool {
-		res = append(res, h)
-		return true
-	})
-	return
-}
-
-func intersectTwoMaps(m1, m2 *kv.HandleMap) *kv.HandleMap {
-	intersected := kv.NewHandleMap()
-	m1.Range(func(h kv.Handle, val interface{}) bool {
-		if _, ok := m2.Get(h); ok {
-			intersected.Set(h, true)
-		}
-		return true
-	})
-	return intersected
-}
-
 func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask) func(r interface{}) {
 	return func(r interface{}) {
 		if r == nil {
diff --git a/executor/utils.go b/executor/utils.go
index 47fe32a93aa68..55ffa4b14b7bb 100644
--- a/executor/utils.go
+++ b/executor/utils.go
@@ -15,6 +15,7 @@
 package executor
 
 import (
+	"github.com/pingcap/tidb/kv"
 	"strings"
 )
 
@@ -92,3 +93,47 @@ func (b *batchRetrieverHelper) nextBatch(retrieveRange func(start, end int) erro
 	}
 	return nil
 }
+
+// IntersectHandleMaps get intersection of all handleMaps. Return nil if no intersection.
+func IntersectHandleMaps(handleMaps map[int]*kv.HandleMap) (res []kv.Handle) {
+	if len(handleMaps) == 0 {
+		return nil
+	}
+	resHandleMaps := make([]*kv.HandleMap, 0, len(handleMaps))
+	var gotEmptyHandleMap bool
+	for _, m := range handleMaps {
+		if m.Len() == 0 {
+			gotEmptyHandleMap = true
+		}
+		resHandleMaps = append(resHandleMaps, m)
+	}
+	if gotEmptyHandleMap {
+		return nil
+	}
+
+	intersected := resHandleMaps[0]
+	if len(resHandleMaps) > 1 {
+		for i := 1; i < len(resHandleMaps); i++ {
+			if intersected.Len() == 0 {
+				break
+			}
+			intersected = intersectTwoMaps(intersected, resHandleMaps[i])
+		}
+	}
+	intersected.Range(func(h kv.Handle, val interface{}) bool {
+		res = append(res, h)
+		return true
+	})
+	return
+}
+
+func intersectTwoMaps(m1, m2 *kv.HandleMap) *kv.HandleMap {
+	intersected := kv.NewHandleMap()
+	m1.Range(func(h kv.Handle, val interface{}) bool {
+		if _, ok := m2.Get(h); ok {
+			intersected.Set(h, true)
+		}
+		return true
+	})
+	return intersected
+}
diff --git a/executor/utils_test.go b/executor/utils_test.go
index 3c8a32de25cc5..07112c9768cd7 100644
--- a/executor/utils_test.go
+++ b/executor/utils_test.go
@@ -15,9 +15,11 @@
 package executor
 
 import (
+	"math/rand"
 	"testing"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/tidb/kv"
 	"github.com/stretchr/testify/require"
 )
 
@@ -93,3 +95,38 @@ func TestBatchRetrieverHelper(t *testing.T) {
 	require.Equal(t, rangeStarts, []int{0})
 	require.Equal(t, rangeEnds, []int{10})
 }
+
+func TestIntersectionMaps(t *testing.T) {
+	workerHandleMaps := make(map[int]*kv.HandleMap)
+	var handleOff int
+	workerNum := 10
+	for i := 0; i < workerNum; i++ {
+		m := kv.NewHandleMap()
+		workerHandleMaps[i] = m
+		for j := 0; j < 100; j++ {
+			m.Set(kv.IntHandle(handleOff), true)
+			handleOff++
+		}
+	}
+	res := IntersectHandleMaps(workerHandleMaps)
+	require.Equal(t, 0, len(res))
+
+	for _, m := range workerHandleMaps {
+		m.Set(kv.IntHandle(handleOff), true)
+	}
+	res = IntersectHandleMaps(workerHandleMaps)
+	require.Equal(t, 1, len(res))
+	hitHandleOff := handleOff
+	res[0].Equal(kv.IntHandle(handleOff))
+	handleOff++
+
+	// Pick 3 map to put new handle.
+	for i := 0; i < 3; i++ {
+		m := workerHandleMaps[rand.Intn(workerNum)]
+		m.Set(kv.IntHandle(handleOff), true)
+	}
+	res = IntersectHandleMaps(workerHandleMaps)
+	require.Equal(t, 1, len(res))
+	res[0].Equal(kv.IntHandle(hitHandleOff))
+	handleOff++
+}

From 79af3c072d9c560831ddbd57361a5ee85ad57072 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Thu, 27 Oct 2022 13:41:32 +0800
Subject: [PATCH 05/33] fix lint

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/utils.go      | 3 ++-
 executor/utils_test.go | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/executor/utils.go b/executor/utils.go
index 55ffa4b14b7bb..ae91ed7e2d8be 100644
--- a/executor/utils.go
+++ b/executor/utils.go
@@ -15,8 +15,9 @@
 package executor
 
 import (
-	"github.com/pingcap/tidb/kv"
 	"strings"
+
+	"github.com/pingcap/tidb/kv"
 )
 
 // SetFromString constructs a slice of strings from a comma separated string.
diff --git a/executor/utils_test.go b/executor/utils_test.go
index 07112c9768cd7..3826f76838f49 100644
--- a/executor/utils_test.go
+++ b/executor/utils_test.go
@@ -128,5 +128,4 @@ func TestIntersectionMaps(t *testing.T) {
 	res = IntersectHandleMaps(workerHandleMaps)
 	require.Equal(t, 1, len(res))
 	res[0].Equal(kv.IntHandle(hitHandleOff))
-	handleOff++
 }

From e60ce706623b22aee9050a1be1a6835a89a891c4 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 21 Nov 2022 22:51:32 +0800
Subject: [PATCH 06/33] add mem_reader

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 147 ++++++++++++++++-----------------
 executor/mem_reader.go         |  53 +++++++++++-
 2 files changed, 121 insertions(+), 79 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index b61ffbfa76be5..7a96f8f69eeec 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"reflect"
 	"runtime/trace"
 	"sync"
 	"sync/atomic"
@@ -793,6 +794,27 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 	}
 }
 
+type processWorker struct {
+	handleMap  *kv.HandleMap
+	workerCh   chan *indexMergeTableTask
+	indexMerge *IndexMergeReaderExecutor
+	wg         *sync.WaitGroup
+}
+
+func (w *processWorker) doIntersectionPerPartition() {
+	for task := range w.workerCh {
+		for _, h := range task.handles {
+			if cntPtr, ok := w.handleMap.Get(h); ok {
+				(*cntPtr.(*int))++
+			} else {
+				cnt := 1
+				w.handleMap.Set(h, &cnt)
+			}
+		}
+	}
+	w.wg.Done()
+}
+
 func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
 	workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
 	defer func() {
@@ -800,96 +822,67 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		close(resultCh)
 	}()
 
-	// key: handle, value: partIdx for this handle.
-	partHandleMap := kv.NewHandleMap()
-	// key: partialWorker, value: handles for this partialWorker.
-	workerHandleMap := make(map[int]*kv.HandleMap)
-	for i := 0; i < len(w.indexMerge.partialPlans); i++ {
-		workerHandleMap[i] = kv.NewHandleMap()
-	}
-	// Distinct handles for each partialWorker.
-	for task := range fetchCh {
+	if w.stats != nil {
 		start := time.Now()
-		handles := task.handles
+		defer func() {
+			w.stats.IndexMergeProcess += time.Since(start)
+		}()
+	}
 
-		if _, ok := workerHandleMap[task.workerID]; !ok {
-			err := errors.Errorf("unexpected workerID, got %d, total: %d", task.workerID, len(w.indexMerge.partialPlans))
-			syncErr(resultCh, err)
-			return
-		}
-		hMap := workerHandleMap[task.workerID]
+	mapLen := 1
+	if w.indexMerge.partitionTableMode {
+		mapLen = len(w.indexMerge.prunedPartitions)
+	}
+	handleMaps := make([]*kv.HandleMap, 0, mapLen)
+	workers := make([]*processWorker, 0, len(handleMaps))
+	wg := sync.WaitGroup{}
+	for i := 0; i < mapLen; i++ {
+		handleMaps = append(handleMaps, kv.NewHandleMap())
+		workers = append(workers, &processWorker{
+			handleMap:  handleMaps[i],
+			workerCh:   make(chan *indexMergeTableTask, 10000),
+			indexMerge: w.indexMerge,
+			wg:         &wg,
+		})
+		go workers[i].doIntersectionPerPartition()
+		wg.Add(1)
+	}
 
-		for _, h := range handles {
-			if _, ok := hMap.Get(h); !ok {
-				hMap.Set(h, true)
-			}
-			if w.indexMerge.partitionTableMode {
-				if parTblIdx, ok := partHandleMap.Get(h); ok {
-					if parTblIdx.(int) != task.parTblIdx {
-						err := errors.Errorf("expected parTblIdx: %v, got: %d", parTblIdx, task.parTblIdx)
-						syncErr(resultCh, err)
-						return
-					}
-				} else {
-					partHandleMap.Set(h, task.parTblIdx)
-				}
-			}
-		}
-		if w.stats != nil {
-			w.stats.IndexMergeProcess += time.Since(start)
-		}
+	for task := range fetchCh {
+		workers[task.parTblIdx].workerCh <- task
+	}
+	for _, w := range workers {
+		close(w.workerCh)
+	}
+	wg.Wait()
+	for _, w := range workers {
+		w.indexMerge.memTracker.Consume(int64(w.handleMap.Len()) * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
 	}
 
-	start := time.Now()
-	defer func() {
-		if w.stats != nil {
-			w.stats.IndexMergeProcess += time.Since(start)
-		}
-	}()
-	// Intersect handles among all partialWorkers.
-	intersected := IntersectHandleMaps(workerHandleMap)
-	if len(intersected) == 0 {
-		// No data to send, just return, resultCh and workCh will be closed in defer func.
-		return
+	intersected := make([][]kv.Handle, len(handleMaps))
+	for parTblIdx, m := range handleMaps {
+		m.Range(func(h kv.Handle, val interface{}) bool {
+			if *(val.(*int)) == len(w.indexMerge.partialPlans) {
+				// Means all partial paths have this handle.
+				intersected[parTblIdx] = append(intersected[parTblIdx], h)
+			}
+			return true
+		})
 	}
-	var tasks []*indexMergeTableTask
-	if !w.indexMerge.partitionTableMode {
+
+	tasks := make([]*indexMergeTableTask, 0, len(handleMaps))
+	for i := 0; i < len(handleMaps); i++ {
 		tasks = append(tasks, &indexMergeTableTask{
 			lookupTableTask: lookupTableTask{
-				handles: intersected,
+				handles: intersected[i],
 				doneCh:  make(chan error, 1),
 			},
 		})
-	} else {
-		// Need to distinguish which parition table this handle belongs to.
-		// key: partIdx, value: intersected handles of this partIdx.
-		intersectedPartHandleMap := make(map[int][]kv.Handle)
-		for _, h := range intersected {
-			parTblIdxI, ok := partHandleMap.Get(h)
-			if !ok {
-				err := errors.New("handle not found")
-				syncErr(resultCh, err)
-				return
-			}
-			parTblIdx := parTblIdxI.(int)
-
-			if _, ok := intersectedPartHandleMap[parTblIdx]; !ok {
-				intersectedPartHandleMap[parTblIdx] = make([]kv.Handle, 0, 8)
-			}
-			intersectedPartHandleMap[parTblIdx] = append(intersectedPartHandleMap[parTblIdx], h)
-		}
-
-		for parTblIdx, handles := range intersectedPartHandleMap {
-			tasks = append(tasks, &indexMergeTableTask{
-				lookupTableTask: lookupTableTask{
-					handles: handles,
-					doneCh:  make(chan error, 1),
-
-					partitionTable: w.indexMerge.prunedPartitions[parTblIdx],
-				},
-			})
+		if w.indexMerge.partitionTableMode {
+			tasks[i].partitionTable = w.indexMerge.prunedPartitions[i]
 		}
 	}
+
 	for _, task := range tasks {
 		select {
 		case <-ctx.Done():
diff --git a/executor/mem_reader.go b/executor/mem_reader.go
index 3790a5d862539..2e63af9a1b74f 100644
--- a/executor/mem_reader.go
+++ b/executor/mem_reader.go
@@ -595,6 +595,7 @@ type memIndexMergeReader struct {
 	retFieldTypes    []*types.FieldType
 	indexMergeReader *IndexMergeReaderExecutor
 	memReaders       []memReader
+	isIntersection   bool
 
 	// partition mode
 	partitionMode     bool                  // if it is accessing a partition table
@@ -651,6 +652,7 @@ func buildMemIndexMergeReader(ctx context.Context, us *UnionScanExec, indexMerge
 		retFieldTypes:    retTypes(us),
 		indexMergeReader: indexMergeReader,
 		memReaders:       memReaders,
+		isIntersection:   indexMergeReader.isIntersection,
 
 		partitionMode:     indexMergeReader.partitionTableMode,
 		partitionTables:   indexMergeReader.prunedPartitions,
@@ -676,11 +678,20 @@ func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum,
 	} else {
 		kvRanges = append(kvRanges, m.indexMergeReader.keyRanges)
 	}
+	if len(kvRanges) != len(tbls) {
+		return nil, errors.Errorf("unexpected length of tbls(size: %d) should be equals to kvRanges(size: %d)", len(tbls), len(kvRanges))
+	}
 
 	tblKVRanges := make([]kv.KeyRange, 0, 16)
 	numHandles := 0
+	var handles []kv.Handle
+	var err error
 	for i, tbl := range tbls {
-		handles, err := m.unionHandles(kvRanges[i])
+		if m.isIntersection {
+			handles, err = m.intersectionHandles(kvRanges[i])
+		} else {
+			handles, err = m.unionHandles(kvRanges[i])
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -716,7 +727,7 @@ func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum,
 	return memTblReader.getMemRows(ctx)
 }
 
-// Union all handles of different Indexes.
+// Union all handles of all partial paths.
 func (m *memIndexMergeReader) unionHandles(kvRanges [][]kv.KeyRange) (finalHandles []kv.Handle, err error) {
 	if len(m.memReaders) != len(kvRanges) {
 		return nil, errors.Errorf("len(kvRanges) should be equal to len(memReaders)")
@@ -747,6 +758,44 @@ func (m *memIndexMergeReader) unionHandles(kvRanges [][]kv.KeyRange) (finalHandl
 	return finalHandles, nil
 }
 
+// Intersect all handles of all partial paths.
+func (m *memIndexMergeReader) intersectionHandles(kvRanges [][]kv.KeyRange) (finalHandles []kv.Handle, err error) {
+	if len(m.memReaders) != len(kvRanges) {
+		return nil, errors.Errorf("len(kvRanges) should be equal to len(memReaders)")
+	}
+
+	hMap := kv.NewHandleMap()
+	var handles []kv.Handle
+	for i, reader := range m.memReaders {
+		switch r := reader.(type) {
+		case *memTableReader:
+			r.kvRanges = kvRanges[i]
+		case *memIndexReader:
+			r.kvRanges = kvRanges[i]
+		default:
+			return nil, errors.New("memReader have to be memTableReader or memIndexReader")
+		}
+		if handles, err = reader.getMemRowsHandle(); err != nil {
+			return nil, err
+		}
+		for _, h := range handles {
+			if cntPtr, ok := hMap.Get(h); !ok {
+				cnt := 1
+				hMap.Set(h, &cnt)
+			} else {
+				*(cntPtr.(*int)) += 1
+			}
+		}
+	}
+	hMap.Range(func(h kv.Handle, val interface{}) bool {
+		if *(val.(*int)) == len(m.memReaders) {
+			finalHandles = append(finalHandles, h)
+		}
+		return true
+	})
+	return finalHandles, nil
+}
+
 func (m *memIndexMergeReader) getMemRowsHandle() ([]kv.Handle, error) {
 	return nil, errors.New("getMemRowsHandle has not been implemented for memIndexMergeReader")
 }

From 425a7b27e3610e089096a8de0bc01be138b012ab Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 22 Nov 2022 15:24:10 +0800
Subject: [PATCH 07/33] refine processWorker

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 7a96f8f69eeec..c279e5e6b07d7 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -799,10 +799,12 @@ type processWorker struct {
 	workerCh   chan *indexMergeTableTask
 	indexMerge *IndexMergeReaderExecutor
 	wg         *sync.WaitGroup
+	memTracker *memory.Tracker
 }
 
 func (w *processWorker) doIntersectionPerPartition() {
 	for task := range w.workerCh {
+		oriCnt := w.handleMap.Len()
 		for _, h := range task.handles {
 			if cntPtr, ok := w.handleMap.Get(h); ok {
 				(*cntPtr.(*int))++
@@ -811,6 +813,10 @@ func (w *processWorker) doIntersectionPerPartition() {
 				w.handleMap.Set(h, &cnt)
 			}
 		}
+		newCnt := w.handleMap.Len()
+		if newCnt > oriCnt {
+			w.memTracker.Consume(int64(newCnt-oriCnt) * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
+		}
 	}
 	w.wg.Done()
 }
@@ -829,6 +835,8 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		}()
 	}
 
+	// mapLen == len(workers) == len(prunedPartitions)
+	// Max number of partition number is 8192. So there will be at most 8192 processWorkers.
 	mapLen := 1
 	if w.indexMerge.partitionTableMode {
 		mapLen = len(w.indexMerge.prunedPartitions)
@@ -840,9 +848,10 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		handleMaps = append(handleMaps, kv.NewHandleMap())
 		workers = append(workers, &processWorker{
 			handleMap:  handleMaps[i],
-			workerCh:   make(chan *indexMergeTableTask, 10000),
+			workerCh:   make(chan *indexMergeTableTask, 1024),
 			indexMerge: w.indexMerge,
 			wg:         &wg,
+			memTracker: w.indexMerge.memTracker,
 		})
 		go workers[i].doIntersectionPerPartition()
 		wg.Add(1)
@@ -855,9 +864,6 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		close(w.workerCh)
 	}
 	wg.Wait()
-	for _, w := range workers {
-		w.indexMerge.memTracker.Consume(int64(w.handleMap.Len()) * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
-	}
 
 	intersected := make([][]kv.Handle, len(handleMaps))
 	for parTblIdx, m := range handleMaps {

From 2b450b4b2396ab2d9357ebd644178a36efda4c4b Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 22 Nov 2022 16:29:33 +0800
Subject: [PATCH 08/33] refine intersectionProcessWorker

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 78 +++++++++++++++++++++-------------
 1 file changed, 49 insertions(+), 29 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index c279e5e6b07d7..3513b65b00208 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -794,26 +794,28 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 	}
 }
 
-type processWorker struct {
-	handleMap  *kv.HandleMap
+type intersectionProcessWorker struct {
+	// key: parTblIdx, val: HandleMap
+	handleMaps map[int]*kv.HandleMap
 	workerCh   chan *indexMergeTableTask
 	indexMerge *IndexMergeReaderExecutor
 	wg         *sync.WaitGroup
 	memTracker *memory.Tracker
 }
 
-func (w *processWorker) doIntersectionPerPartition() {
+func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 	for task := range w.workerCh {
-		oriCnt := w.handleMap.Len()
+		hMap := w.handleMaps[task.parTblIdx]
+		oriCnt := hMap.Len()
 		for _, h := range task.handles {
-			if cntPtr, ok := w.handleMap.Get(h); ok {
+			if cntPtr, ok := hMap.Get(h); ok {
 				(*cntPtr.(*int))++
 			} else {
 				cnt := 1
-				w.handleMap.Set(h, &cnt)
+				hMap.Set(h, &cnt)
 			}
 		}
-		newCnt := w.handleMap.Len()
+		newCnt := hMap.Len()
 		if newCnt > oriCnt {
 			w.memTracker.Consume(int64(newCnt-oriCnt) * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
 		}
@@ -835,20 +837,38 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		}()
 	}
 
-	// mapLen == len(workers) == len(prunedPartitions)
-	// Max number of partition number is 8192. So there will be at most 8192 processWorkers.
-	mapLen := 1
+	partCnt := 1
+	workerCnt := 1
+	partCntPerWorker := 1
+	// Use one goroutine to handle multiple partitions.
+	// Max number of partition number is 8192, we use 64 to avoid too many goroutines.
+	const maxWorkerCnt int = 64
+	const maxChannelSize int = 1024
 	if w.indexMerge.partitionTableMode {
-		mapLen = len(w.indexMerge.prunedPartitions)
+		partCnt = len(w.indexMerge.prunedPartitions)
+		workerCnt = partCnt
+		if workerCnt > maxWorkerCnt {
+			workerCnt = maxWorkerCnt
+			partCntPerWorker = partCnt / workerCnt
+			if partCnt%workerCnt != 0 {
+				partCntPerWorker++
+			}
+		}
 	}
-	handleMaps := make([]*kv.HandleMap, 0, mapLen)
-	workers := make([]*processWorker, 0, len(handleMaps))
+	handleMaps := make([]*kv.HandleMap, 0, partCnt)
+	workers := make([]*intersectionProcessWorker, 0, workerCnt)
 	wg := sync.WaitGroup{}
-	for i := 0; i < mapLen; i++ {
-		handleMaps = append(handleMaps, kv.NewHandleMap())
-		workers = append(workers, &processWorker{
-			handleMap:  handleMaps[i],
-			workerCh:   make(chan *indexMergeTableTask, 1024),
+	for i := 0; i < workerCnt; i++ {
+		handleMapsPerWorker := make(map[int]*kv.HandleMap, partCntPerWorker)
+		for j := 0; j < partCntPerWorker; j++ {
+			hMap := kv.NewHandleMap()
+			handleMaps = append(handleMaps, hMap)
+			parTblIdx := i*partCntPerWorker + j
+			handleMapsPerWorker[parTblIdx] = hMap
+		}
+		workers = append(workers, &intersectionProcessWorker{
+			handleMaps: handleMapsPerWorker,
+			workerCh:   make(chan *indexMergeTableTask, maxChannelSize),
 			indexMerge: w.indexMerge,
 			wg:         &wg,
 			memTracker: w.indexMerge.memTracker,
@@ -856,18 +876,17 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		go workers[i].doIntersectionPerPartition()
 		wg.Add(1)
 	}
-
 	for task := range fetchCh {
-		workers[task.parTblIdx].workerCh <- task
+		workers[task.parTblIdx%maxWorkerCnt].workerCh <- task
 	}
 	for _, w := range workers {
 		close(w.workerCh)
 	}
 	wg.Wait()
 
-	intersected := make([][]kv.Handle, len(handleMaps))
-	for parTblIdx, m := range handleMaps {
-		m.Range(func(h kv.Handle, val interface{}) bool {
+	intersected := make([][]kv.Handle, 0, partCnt)
+	for parTblIdx, hMap := range handleMaps {
+		hMap.Range(func(h kv.Handle, val interface{}) bool {
 			if *(val.(*int)) == len(w.indexMerge.partialPlans) {
 				// Means all partial paths have this handle.
 				intersected[parTblIdx] = append(intersected[parTblIdx], h)
@@ -875,20 +894,21 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			return true
 		})
 	}
-
-	tasks := make([]*indexMergeTableTask, 0, len(handleMaps))
-	for i := 0; i < len(handleMaps); i++ {
+	tasks := make([]*indexMergeTableTask, 0, partCnt)
+	for parTblIdx := 0; parTblIdx < partCnt; parTblIdx++ {
+		if len(intersected[parTblIdx]) == 0 {
+			continue
+		}
 		tasks = append(tasks, &indexMergeTableTask{
 			lookupTableTask: lookupTableTask{
-				handles: intersected[i],
+				handles: intersected[parTblIdx],
 				doneCh:  make(chan error, 1),
 			},
 		})
 		if w.indexMerge.partitionTableMode {
-			tasks[i].partitionTable = w.indexMerge.prunedPartitions[i]
+			tasks[parTblIdx].partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
 		}
 	}
-
 	for _, task := range tasks {
 		select {
 		case <-ctx.Done():

From 816b378b5075379fbafe46295653dec322e6b55b Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 22 Nov 2022 19:37:51 +0800
Subject: [PATCH 09/33] refine intersectionProcessWorker

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 3513b65b00208..ed975fbb9c217 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -795,8 +795,9 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 }
 
 type intersectionProcessWorker struct {
+	workerID int
 	// key: parTblIdx, val: HandleMap
-	handleMaps map[int]*kv.HandleMap
+	handleMapsPerWorker map[int]*kv.HandleMap
 	workerCh   chan *indexMergeTableTask
 	indexMerge *IndexMergeReaderExecutor
 	wg         *sync.WaitGroup
@@ -805,7 +806,11 @@ type intersectionProcessWorker struct {
 
 func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 	for task := range w.workerCh {
-		hMap := w.handleMaps[task.parTblIdx]
+		var ok bool
+		var hMap *kv.HandleMap
+		if hMap, ok = w.handleMapsPerWorker[task.parTblIdx]; !ok {
+			panic(fmt.Sprintf("cannot find parTblIdx(%d) for worker(id: %d)", task.parTblIdx, w.workerID))
+		}
 		oriCnt := hMap.Len()
 		for _, h := range task.handles {
 			if cntPtr, ok := hMap.Get(h); ok {
@@ -840,10 +845,10 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	partCnt := 1
 	workerCnt := 1
 	partCntPerWorker := 1
-	// Use one goroutine to handle multiple partitions.
-	// Max number of partition number is 8192, we use 64 to avoid too many goroutines.
-	const maxWorkerCnt int = 64
-	const maxChannelSize int = 1024
+	// One goroutine may handle one or multiple partitions.
+	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
+	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().ExecutorConcurrency
+	maxChannelSize := 1024
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
 		workerCnt = partCnt
@@ -867,7 +872,8 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			handleMapsPerWorker[parTblIdx] = hMap
 		}
 		workers = append(workers, &intersectionProcessWorker{
-			handleMaps: handleMapsPerWorker,
+			workerID: i,
+			handleMapsPerWorker: handleMapsPerWorker,
 			workerCh:   make(chan *indexMergeTableTask, maxChannelSize),
 			indexMerge: w.indexMerge,
 			wg:         &wg,
@@ -877,14 +883,14 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		wg.Add(1)
 	}
 	for task := range fetchCh {
-		workers[task.parTblIdx%maxWorkerCnt].workerCh <- task
+		workers[task.parTblIdx/partCntPerWorker].workerCh <- task
 	}
 	for _, w := range workers {
 		close(w.workerCh)
 	}
 	wg.Wait()
 
-	intersected := make([][]kv.Handle, 0, partCnt)
+	intersected := make([][]kv.Handle, partCnt)
 	for parTblIdx, hMap := range handleMaps {
 		hMap.Range(func(h kv.Handle, val interface{}) bool {
 			if *(val.(*int)) == len(w.indexMerge.partialPlans) {

From 096dfdd1bcb8cad473c26c0cb113589691832ec4 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Wed, 23 Nov 2022 11:02:23 +0800
Subject: [PATCH 10/33] fix fmt

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index ed975fbb9c217..07cb6329c605d 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -798,10 +798,10 @@ type intersectionProcessWorker struct {
 	workerID int
 	// key: parTblIdx, val: HandleMap
 	handleMapsPerWorker map[int]*kv.HandleMap
-	workerCh   chan *indexMergeTableTask
-	indexMerge *IndexMergeReaderExecutor
-	wg         *sync.WaitGroup
-	memTracker *memory.Tracker
+	workerCh            chan *indexMergeTableTask
+	indexMerge          *IndexMergeReaderExecutor
+	wg                  *sync.WaitGroup
+	memTracker          *memory.Tracker
 }
 
 func (w *intersectionProcessWorker) doIntersectionPerPartition() {
@@ -872,12 +872,12 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			handleMapsPerWorker[parTblIdx] = hMap
 		}
 		workers = append(workers, &intersectionProcessWorker{
-			workerID: i,
+			workerID:            i,
 			handleMapsPerWorker: handleMapsPerWorker,
-			workerCh:   make(chan *indexMergeTableTask, maxChannelSize),
-			indexMerge: w.indexMerge,
-			wg:         &wg,
-			memTracker: w.indexMerge.memTracker,
+			workerCh:            make(chan *indexMergeTableTask, maxChannelSize),
+			indexMerge:          w.indexMerge,
+			wg:                  &wg,
+			memTracker:          w.indexMerge.memTracker,
 		})
 		go workers[i].doIntersectionPerPartition()
 		wg.Add(1)

From 742dbf3d1667a4f7010e276000eac2bffdd9b62c Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Wed, 23 Nov 2022 11:27:53 +0800
Subject: [PATCH 11/33] fix comment

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/mem_reader.go |  4 ++--
 executor/utils.go      | 46 ------------------------------------------
 executor/utils_test.go | 34 -------------------------------
 3 files changed, 2 insertions(+), 82 deletions(-)

diff --git a/executor/mem_reader.go b/executor/mem_reader.go
index 2e63af9a1b74f..647d785caeafc 100644
--- a/executor/mem_reader.go
+++ b/executor/mem_reader.go
@@ -679,7 +679,7 @@ func (m *memIndexMergeReader) getMemRows(ctx context.Context) ([][]types.Datum,
 		kvRanges = append(kvRanges, m.indexMergeReader.keyRanges)
 	}
 	if len(kvRanges) != len(tbls) {
-		return nil, errors.Errorf("unexpected length of tbls(size: %d) should be equals to kvRanges(size: %d)", len(tbls), len(kvRanges))
+		return nil, errors.Errorf("length of tbls(size: %d) should be equals to length of kvRanges(size: %d)", len(tbls), len(kvRanges))
 	}
 
 	tblKVRanges := make([]kv.KeyRange, 0, 16)
@@ -758,7 +758,7 @@ func (m *memIndexMergeReader) unionHandles(kvRanges [][]kv.KeyRange) (finalHandl
 	return finalHandles, nil
 }
 
-// Intersect all handles of all partial paths.
+// Intersect handles of each partial paths.
 func (m *memIndexMergeReader) intersectionHandles(kvRanges [][]kv.KeyRange) (finalHandles []kv.Handle, err error) {
 	if len(m.memReaders) != len(kvRanges) {
 		return nil, errors.Errorf("len(kvRanges) should be equal to len(memReaders)")
diff --git a/executor/utils.go b/executor/utils.go
index ae91ed7e2d8be..47fe32a93aa68 100644
--- a/executor/utils.go
+++ b/executor/utils.go
@@ -16,8 +16,6 @@ package executor
 
 import (
 	"strings"
-
-	"github.com/pingcap/tidb/kv"
 )
 
 // SetFromString constructs a slice of strings from a comma separated string.
@@ -94,47 +92,3 @@ func (b *batchRetrieverHelper) nextBatch(retrieveRange func(start, end int) erro
 	}
 	return nil
 }
-
-// IntersectHandleMaps get intersection of all handleMaps. Return nil if no intersection.
-func IntersectHandleMaps(handleMaps map[int]*kv.HandleMap) (res []kv.Handle) {
-	if len(handleMaps) == 0 {
-		return nil
-	}
-	resHandleMaps := make([]*kv.HandleMap, 0, len(handleMaps))
-	var gotEmptyHandleMap bool
-	for _, m := range handleMaps {
-		if m.Len() == 0 {
-			gotEmptyHandleMap = true
-		}
-		resHandleMaps = append(resHandleMaps, m)
-	}
-	if gotEmptyHandleMap {
-		return nil
-	}
-
-	intersected := resHandleMaps[0]
-	if len(resHandleMaps) > 1 {
-		for i := 1; i < len(resHandleMaps); i++ {
-			if intersected.Len() == 0 {
-				break
-			}
-			intersected = intersectTwoMaps(intersected, resHandleMaps[i])
-		}
-	}
-	intersected.Range(func(h kv.Handle, val interface{}) bool {
-		res = append(res, h)
-		return true
-	})
-	return
-}
-
-func intersectTwoMaps(m1, m2 *kv.HandleMap) *kv.HandleMap {
-	intersected := kv.NewHandleMap()
-	m1.Range(func(h kv.Handle, val interface{}) bool {
-		if _, ok := m2.Get(h); ok {
-			intersected.Set(h, true)
-		}
-		return true
-	})
-	return intersected
-}
diff --git a/executor/utils_test.go b/executor/utils_test.go
index 3826f76838f49..07b824569b4e4 100644
--- a/executor/utils_test.go
+++ b/executor/utils_test.go
@@ -95,37 +95,3 @@ func TestBatchRetrieverHelper(t *testing.T) {
 	require.Equal(t, rangeStarts, []int{0})
 	require.Equal(t, rangeEnds, []int{10})
 }
-
-func TestIntersectionMaps(t *testing.T) {
-	workerHandleMaps := make(map[int]*kv.HandleMap)
-	var handleOff int
-	workerNum := 10
-	for i := 0; i < workerNum; i++ {
-		m := kv.NewHandleMap()
-		workerHandleMaps[i] = m
-		for j := 0; j < 100; j++ {
-			m.Set(kv.IntHandle(handleOff), true)
-			handleOff++
-		}
-	}
-	res := IntersectHandleMaps(workerHandleMaps)
-	require.Equal(t, 0, len(res))
-
-	for _, m := range workerHandleMaps {
-		m.Set(kv.IntHandle(handleOff), true)
-	}
-	res = IntersectHandleMaps(workerHandleMaps)
-	require.Equal(t, 1, len(res))
-	hitHandleOff := handleOff
-	res[0].Equal(kv.IntHandle(handleOff))
-	handleOff++
-
-	// Pick 3 map to put new handle.
-	for i := 0; i < 3; i++ {
-		m := workerHandleMaps[rand.Intn(workerNum)]
-		m.Set(kv.IntHandle(handleOff), true)
-	}
-	res = IntersectHandleMaps(workerHandleMaps)
-	require.Equal(t, 1, len(res))
-	res[0].Equal(kv.IntHandle(hitHandleOff))
-}

From 0addbf5c5ed8f5af171140c06477ea8dba460115 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Wed, 23 Nov 2022 12:02:13 +0800
Subject: [PATCH 12/33] fix

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/utils_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/executor/utils_test.go b/executor/utils_test.go
index 07b824569b4e4..3c8a32de25cc5 100644
--- a/executor/utils_test.go
+++ b/executor/utils_test.go
@@ -15,11 +15,9 @@
 package executor
 
 import (
-	"math/rand"
 	"testing"
 
 	"github.com/pingcap/errors"
-	"github.com/pingcap/tidb/kv"
 	"github.com/stretchr/testify/require"
 )
 

From 6e9720409d9799660dee1a58032dda59fd4f51e6 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Thu, 24 Nov 2022 17:34:51 +0800
Subject: [PATCH 13/33] fix panic

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 07cb6329c605d..5d3319787f38b 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -912,7 +912,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			},
 		})
 		if w.indexMerge.partitionTableMode {
-			tasks[parTblIdx].partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
+			tasks[len(tasks)-1].partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
 		}
 	}
 	for _, task := range tasks {

From 1c2e2747fe0a638ed318c6ee4bb426b6ffad6bef Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Fri, 25 Nov 2022 11:07:58 +0800
Subject: [PATCH 14/33] fix fetchCh size

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 5d3319787f38b..b421d2a061902 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -222,7 +222,7 @@ func (e *IndexMergeReaderExecutor) buildKeyRangesForTable(tbl table.Table) (rang
 func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
 	exitCh := make(chan struct{})
 	workCh := make(chan *indexMergeTableTask, 1)
-	fetchCh := make(chan *indexMergeTableTask, len(e.keyRanges))
+	fetchCh := make(chan *indexMergeTableTask, len(e.partialPlans))
 
 	e.startIndexMergeProcessWorker(ctx, workCh, fetchCh)
 

From 4b92b7171dd479345513101e9307ca74dfb4bea9 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Fri, 25 Nov 2022 13:18:50 +0800
Subject: [PATCH 15/33] add tidb_index_merge_intersection_concurrency

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go       |  10 +-
 sessionctx/variable/session.go       |  40 ++-
 sessionctx/variable/sysvar.go        |   7 +
 sessionctx/variable/tidb_vars.go     | 416 ++++++++++++++-------------
 sessionctx/variable/varsutil_test.go |   1 +
 5 files changed, 254 insertions(+), 220 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index b421d2a061902..12288ccaf7c19 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -222,7 +222,7 @@ func (e *IndexMergeReaderExecutor) buildKeyRangesForTable(tbl table.Table) (rang
 func (e *IndexMergeReaderExecutor) startWorkers(ctx context.Context) error {
 	exitCh := make(chan struct{})
 	workCh := make(chan *indexMergeTableTask, 1)
-	fetchCh := make(chan *indexMergeTableTask, len(e.partialPlans))
+	fetchCh := make(chan *indexMergeTableTask, len(e.keyRanges))
 
 	e.startIndexMergeProcessWorker(ctx, workCh, fetchCh)
 
@@ -847,7 +847,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	partCntPerWorker := 1
 	// One goroutine may handle one or multiple partitions.
 	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
-	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().ExecutorConcurrency
+	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
 	maxChannelSize := 1024
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
@@ -860,6 +860,12 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			}
 		}
 	}
+	failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
+		con := val.(int)
+		if con != workerCnt {
+			panic(fmt.Sprintf("unexpected workerCnt, expect %d, got %d", con, workerCnt))
+		}
+	})
 	handleMaps := make([]*kv.HandleMap, 0, partCnt)
 	workers := make([]*intersectionProcessWorker, 0, workerCnt)
 	wg := sync.WaitGroup{}
diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go
index b8fbcf54848e1..29b96115f8393 100644
--- a/sessionctx/variable/session.go
+++ b/sessionctx/variable/session.go
@@ -1660,18 +1660,19 @@ func NewSessionVars(hctx HookContext) *SessionVars {
 	}
 	vars.KVVars = tikvstore.NewVariables(&vars.Killed)
 	vars.Concurrency = Concurrency{
-		indexLookupConcurrency:     DefIndexLookupConcurrency,
-		indexSerialScanConcurrency: DefIndexSerialScanConcurrency,
-		indexLookupJoinConcurrency: DefIndexLookupJoinConcurrency,
-		hashJoinConcurrency:        DefTiDBHashJoinConcurrency,
-		projectionConcurrency:      DefTiDBProjectionConcurrency,
-		distSQLScanConcurrency:     DefDistSQLScanConcurrency,
-		hashAggPartialConcurrency:  DefTiDBHashAggPartialConcurrency,
-		hashAggFinalConcurrency:    DefTiDBHashAggFinalConcurrency,
-		windowConcurrency:          DefTiDBWindowConcurrency,
-		mergeJoinConcurrency:       DefTiDBMergeJoinConcurrency,
-		streamAggConcurrency:       DefTiDBStreamAggConcurrency,
-		ExecutorConcurrency:        DefExecutorConcurrency,
+		indexLookupConcurrency:            DefIndexLookupConcurrency,
+		indexSerialScanConcurrency:        DefIndexSerialScanConcurrency,
+		indexLookupJoinConcurrency:        DefIndexLookupJoinConcurrency,
+		hashJoinConcurrency:               DefTiDBHashJoinConcurrency,
+		projectionConcurrency:             DefTiDBProjectionConcurrency,
+		distSQLScanConcurrency:            DefDistSQLScanConcurrency,
+		hashAggPartialConcurrency:         DefTiDBHashAggPartialConcurrency,
+		hashAggFinalConcurrency:           DefTiDBHashAggFinalConcurrency,
+		windowConcurrency:                 DefTiDBWindowConcurrency,
+		mergeJoinConcurrency:              DefTiDBMergeJoinConcurrency,
+		streamAggConcurrency:              DefTiDBStreamAggConcurrency,
+		indexMergeIntersectionConcurrency: DefTiDBIndexMergeIntersectionConcurrency,
+		ExecutorConcurrency:               DefExecutorConcurrency,
 	}
 	vars.MemQuota = MemQuota{
 		MemQuotaQuery:      DefTiDBMemQuotaQuery,
@@ -2386,6 +2387,10 @@ type Concurrency struct {
 	// streamAggConcurrency is deprecated, use ExecutorConcurrency instead.
 	streamAggConcurrency int
 
+	// indexMergeIntersectionConcurrency is the number of indexMergeProcessWorker
+	// Only meaningful for dynamic pruned partition table.
+	indexMergeIntersectionConcurrency int
+
 	// indexSerialScanConcurrency is the number of concurrent index serial scan worker.
 	indexSerialScanConcurrency int
 
@@ -2446,6 +2451,10 @@ func (c *Concurrency) SetStreamAggConcurrency(n int) {
 	c.streamAggConcurrency = n
 }
 
+func (c *Concurrency) SetIndexMergeIntersectionConcurrency(n int) {
+	c.indexMergeIntersectionConcurrency = n
+}
+
 // SetIndexSerialScanConcurrency set the number of concurrent index serial scan worker.
 func (c *Concurrency) SetIndexSerialScanConcurrency(n int) {
 	c.indexSerialScanConcurrency = n
@@ -2528,6 +2537,13 @@ func (c *Concurrency) StreamAggConcurrency() int {
 	return c.ExecutorConcurrency
 }
 
+func (c *Concurrency) IndexMergeIntersectionConcurrency() int {
+	if c.indexMergeIntersectionConcurrency != ConcurrencyUnset {
+		return c.indexMergeIntersectionConcurrency
+	}
+	return c.ExecutorConcurrency
+}
+
 // IndexSerialScanConcurrency return the number of concurrent index serial scan worker.
 // This option is not sync with ExecutorConcurrency since it's used by Analyze table.
 func (c *Concurrency) IndexSerialScanConcurrency() int {
diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go
index b3902d8f0e431..89301a162beb2 100644
--- a/sessionctx/variable/sysvar.go
+++ b/sessionctx/variable/sysvar.go
@@ -1507,6 +1507,13 @@ var defaultSysVars = []*SysVar{
 		appendDeprecationWarning(vars, TiDBStreamAggConcurrency, TiDBExecutorConcurrency)
 		return normalizedValue, nil
 	}},
+	{Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexMergeIntersectionConcurrency, Value: strconv.Itoa(DefTiDBIndexMergeIntersectionConcurrency), Type: TypeInt, MinValue: 1, MaxValue: MaxConfigurableConcurrency, AllowAutoValue: true, SetSession: func(s *SessionVars, val string) error {
+		s.indexMergeIntersectionConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset)
+		return nil
+	}, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) {
+		appendDeprecationWarning(vars, TiDBIndexMergeIntersectionConcurrency, TiDBExecutorConcurrency)
+		return normalizedValue, nil
+	}},
 	{Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableParallelApply, Value: BoolToOnOff(DefTiDBEnableParallelApply), Type: TypeBool, SetSession: func(s *SessionVars, val string) error {
 		s.EnableParallelApply = TiDBOptOn(val)
 		return nil
diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go
index 3511775de08f1..9b5a003ddf2f1 100644
--- a/sessionctx/variable/tidb_vars.go
+++ b/sessionctx/variable/tidb_vars.go
@@ -425,6 +425,9 @@ const (
 	// tidb_stream_agg_concurrency is deprecated, use tidb_executor_concurrency instead.
 	TiDBStreamAggConcurrency = "tidb_streamagg_concurrency"
 
+	// TiDBIndexMergeIntersectionConcurrency is used for parallel worker of index merge intersection.
+	TiDBIndexMergeIntersectionConcurrency = "tidb_index_merge_intersection_concurrency"
+
 	// TiDBEnableParallelApply is used for parallel apply.
 	TiDBEnableParallelApply = "tidb_enable_parallel_apply"
 
@@ -865,212 +868,213 @@ const (
 
 // Default TiDB system variable values.
 const (
-	DefHostname                                    = "localhost"
-	DefIndexLookupConcurrency                      = ConcurrencyUnset
-	DefIndexLookupJoinConcurrency                  = ConcurrencyUnset
-	DefIndexSerialScanConcurrency                  = 1
-	DefIndexJoinBatchSize                          = 25000
-	DefIndexLookupSize                             = 20000
-	DefDistSQLScanConcurrency                      = 15
-	DefBuildStatsConcurrency                       = 4
-	DefAutoAnalyzeRatio                            = 0.5
-	DefAutoAnalyzeStartTime                        = "00:00 +0000"
-	DefAutoAnalyzeEndTime                          = "23:59 +0000"
-	DefAutoIncrementIncrement                      = 1
-	DefAutoIncrementOffset                         = 1
-	DefChecksumTableConcurrency                    = 4
-	DefSkipUTF8Check                               = false
-	DefSkipASCIICheck                              = false
-	DefOptAggPushDown                              = false
-	DefOptCartesianBCJ                             = 1
-	DefOptMPPOuterJoinFixedBuildSide               = false
-	DefOptWriteRowID                               = false
-	DefOptEnableCorrelationAdjustment              = true
-	DefOptLimitPushDownThreshold                   = 100
-	DefOptCorrelationThreshold                     = 0.9
-	DefOptCorrelationExpFactor                     = 1
-	DefOptCPUFactor                                = 3.0
-	DefOptCopCPUFactor                             = 3.0
-	DefOptTiFlashConcurrencyFactor                 = 24.0
-	DefOptNetworkFactor                            = 1.0
-	DefOptScanFactor                               = 1.5
-	DefOptDescScanFactor                           = 3.0
-	DefOptSeekFactor                               = 20.0
-	DefOptMemoryFactor                             = 0.001
-	DefOptDiskFactor                               = 1.5
-	DefOptConcurrencyFactor                        = 3.0
-	DefOptForceInlineCTE                           = false
-	DefOptInSubqToJoinAndAgg                       = true
-	DefOptPreferRangeScan                          = false
-	DefBatchInsert                                 = false
-	DefBatchDelete                                 = false
-	DefBatchCommit                                 = false
-	DefCurretTS                                    = 0
-	DefInitChunkSize                               = 32
-	DefMinPagingSize                               = int(paging.MinPagingSize)
-	DefMaxPagingSize                               = int(paging.MaxPagingSize)
-	DefMaxChunkSize                                = 1024
-	DefDMLBatchSize                                = 0
-	DefMaxPreparedStmtCount                        = -1
-	DefWaitTimeout                                 = 28800
-	DefTiDBMemQuotaApplyCache                      = 32 << 20 // 32MB.
-	DefTiDBMemQuotaBindingCache                    = 64 << 20 // 64MB.
-	DefTiDBGeneralLog                              = false
-	DefTiDBPProfSQLCPU                             = 0
-	DefTiDBRetryLimit                              = 10
-	DefTiDBDisableTxnAutoRetry                     = true
-	DefTiDBConstraintCheckInPlace                  = false
-	DefTiDBHashJoinConcurrency                     = ConcurrencyUnset
-	DefTiDBProjectionConcurrency                   = ConcurrencyUnset
-	DefBroadcastJoinThresholdSize                  = 100 * 1024 * 1024
-	DefBroadcastJoinThresholdCount                 = 10 * 1024
-	DefTiDBOptimizerSelectivityLevel               = 0
-	DefTiDBOptimizerEnableNewOFGB                  = false
-	DefTiDBEnableOuterJoinReorder                  = false
-	DefTiDBEnableNAAJ                              = false
-	DefTiDBAllowBatchCop                           = 1
-	DefTiDBAllowMPPExecution                       = true
-	DefTiDBHashExchangeWithNewCollation            = true
-	DefTiDBEnforceMPPExecution                     = false
-	DefTiFlashMaxThreads                           = -1
-	DefTiDBMPPStoreFailTTL                         = "60s"
-	DefTiDBTxnMode                                 = ""
-	DefTiDBRowFormatV1                             = 1
-	DefTiDBRowFormatV2                             = 2
-	DefTiDBDDLReorgWorkerCount                     = 4
-	DefTiDBDDLReorgBatchSize                       = 256
-	DefTiDBDDLFlashbackConcurrency                 = 64
-	DefTiDBDDLErrorCountLimit                      = 512
-	DefTiDBMaxDeltaSchemaCount                     = 1024
-	DefTiDBPlacementMode                           = PlacementModeStrict
-	DefTiDBEnableAutoIncrementInGenerated          = false
-	DefTiDBHashAggPartialConcurrency               = ConcurrencyUnset
-	DefTiDBHashAggFinalConcurrency                 = ConcurrencyUnset
-	DefTiDBWindowConcurrency                       = ConcurrencyUnset
-	DefTiDBMergeJoinConcurrency                    = 1 // disable optimization by default
-	DefTiDBStreamAggConcurrency                    = 1
-	DefTiDBForcePriority                           = mysql.NoPriority
-	DefEnableWindowFunction                        = true
-	DefEnablePipelinedWindowFunction               = true
-	DefEnableStrictDoubleTypeCheck                 = true
-	DefEnableVectorizedExpression                  = true
-	DefTiDBOptJoinReorderThreshold                 = 0
-	DefTiDBDDLSlowOprThreshold                     = 300
-	DefTiDBUseFastAnalyze                          = false
-	DefTiDBSkipIsolationLevelCheck                 = false
-	DefTiDBExpensiveQueryTimeThreshold             = 60 // 60s
-	DefTiDBScatterRegion                           = false
-	DefTiDBWaitSplitRegionFinish                   = true
-	DefWaitSplitRegionTimeout                      = 300 // 300s
-	DefTiDBEnableNoopFuncs                         = Off
-	DefTiDBEnableNoopVariables                     = true
-	DefTiDBAllowRemoveAutoInc                      = false
-	DefTiDBUsePlanBaselines                        = true
-	DefTiDBEvolvePlanBaselines                     = false
-	DefTiDBEvolvePlanTaskMaxTime                   = 600 // 600s
-	DefTiDBEvolvePlanTaskStartTime                 = "00:00 +0000"
-	DefTiDBEvolvePlanTaskEndTime                   = "23:59 +0000"
-	DefInnodbLockWaitTimeout                       = 50 // 50s
-	DefTiDBStoreLimit                              = 0
-	DefTiDBMetricSchemaStep                        = 60 // 60s
-	DefTiDBMetricSchemaRangeDuration               = 60 // 60s
-	DefTiDBFoundInPlanCache                        = false
-	DefTiDBFoundInBinding                          = false
-	DefTiDBEnableCollectExecutionInfo              = true
-	DefTiDBAllowAutoRandExplicitInsert             = false
-	DefTiDBEnableClusteredIndex                    = ClusteredIndexDefModeOn
-	DefTiDBRedactLog                               = false
-	DefTiDBRestrictedReadOnly                      = false
-	DefTiDBSuperReadOnly                           = false
-	DefTiDBShardAllocateStep                       = math.MaxInt64
-	DefTiDBEnableTelemetry                         = true
-	DefTiDBEnableParallelApply                     = false
-	DefTiDBEnableAmendPessimisticTxn               = false
-	DefTiDBPartitionPruneMode                      = "dynamic"
-	DefTiDBEnableRateLimitAction                   = false
-	DefTiDBEnableAsyncCommit                       = false
-	DefTiDBEnable1PC                               = false
-	DefTiDBGuaranteeLinearizability                = true
-	DefTiDBAnalyzeVersion                          = 2
-	DefTiDBAutoAnalyzePartitionBatchSize           = 1
-	DefTiDBEnableIndexMergeJoin                    = false
-	DefTiDBTrackAggregateMemoryUsage               = true
-	DefTiDBEnableExchangePartition                 = true
-	DefCTEMaxRecursionDepth                        = 1000
-	DefTiDBTmpTableMaxSize                         = 64 << 20 // 64MB.
-	DefTiDBEnableLocalTxn                          = false
-	DefTiDBTSOClientBatchMaxWaitTime               = 0.0 // 0ms
-	DefTiDBEnableTSOFollowerProxy                  = false
-	DefTiDBEnableOrderedResultMode                 = false
-	DefTiDBEnablePseudoForOutdatedStats            = false
-	DefTiDBRegardNULLAsPoint                       = true
-	DefEnablePlacementCheck                        = true
-	DefTimestamp                                   = "0"
-	DefTimestampFloat                              = 0.0
-	DefTiDBEnableStmtSummary                       = true
-	DefTiDBStmtSummaryInternalQuery                = false
-	DefTiDBStmtSummaryRefreshInterval              = 1800
-	DefTiDBStmtSummaryHistorySize                  = 24
-	DefTiDBStmtSummaryMaxStmtCount                 = 3000
-	DefTiDBStmtSummaryMaxSQLLength                 = 4096
-	DefTiDBCapturePlanBaseline                     = Off
-	DefTiDBEnableIndexMerge                        = true
-	DefEnableLegacyInstanceScope                   = true
-	DefTiDBTableCacheLease                         = 3 // 3s
-	DefTiDBPersistAnalyzeOptions                   = true
-	DefTiDBEnableColumnTracking                    = false
-	DefTiDBStatsLoadSyncWait                       = 100
-	DefTiDBStatsLoadPseudoTimeout                  = true
-	DefSysdateIsNow                                = false
-	DefTiDBEnableMutationChecker                   = false
-	DefTiDBTxnAssertionLevel                       = AssertionOffStr
-	DefTiDBIgnorePreparedCacheCloseStmt            = false
-	DefTiDBBatchPendingTiFlashCount                = 4000
-	DefRCReadCheckTS                               = false
-	DefTiDBRemoveOrderbyInSubquery                 = false
-	DefTiDBSkewDistinctAgg                         = false
-	DefTiDB3StageDistinctAgg                       = true
-	DefTiDBReadStaleness                           = 0
-	DefTiDBGCMaxWaitTime                           = 24 * 60 * 60
-	DefMaxAllowedPacket                     uint64 = 67108864
-	DefTiDBEnableBatchDML                          = false
-	DefTiDBMemQuotaQuery                           = 1073741824 // 1GB
-	DefTiDBStatsCacheMemQuota                      = 0
-	MaxTiDBStatsCacheMemQuota                      = 1024 * 1024 * 1024 * 1024 // 1TB
-	DefTiDBQueryLogMaxLen                          = 4096
-	DefRequireSecureTransport                      = false
-	DefTiDBCommitterConcurrency                    = 128
-	DefTiDBBatchDMLIgnoreError                     = false
-	DefTiDBMemQuotaAnalyze                         = -1
-	DefTiDBEnableAutoAnalyze                       = true
-	DefTiDBMemOOMAction                            = "CANCEL"
-	DefTiDBMaxAutoAnalyzeTime                      = 12 * 60 * 60
-	DefTiDBEnablePrepPlanCache                     = true
-	DefTiDBPrepPlanCacheSize                       = 100
-	DefTiDBEnablePrepPlanCacheMemoryMonitor        = true
-	DefTiDBPrepPlanCacheMemoryGuardRatio           = 0.1
-	DefTiDBEnableConcurrentDDL                     = concurrencyddl.TiDBEnableConcurrentDDL
-	DefTiDBSimplifiedMetrics                       = false
-	DefTiDBEnablePaging                            = true
-	DefTiFlashFineGrainedShuffleStreamCount        = 0
-	DefStreamCountWhenMaxThreadsNotSet             = 8
-	DefTiFlashFineGrainedShuffleBatchSize          = 8192
-	DefAdaptiveClosestReadThreshold                = 4096
-	DefTiDBEnableAnalyzeSnapshot                   = false
-	DefTiDBGenerateBinaryPlan                      = true
-	DefEnableTiDBGCAwareMemoryTrack                = true
-	DefTiDBDefaultStrMatchSelectivity              = 0.8
-	DefTiDBEnableTmpStorageOnOOM                   = true
-	DefTiDBEnableMDL                               = true
-	DefTiFlashFastScan                             = false
-	DefMemoryUsageAlarmRatio                       = 0.7
-	DefMemoryUsageAlarmKeepRecordNum               = 5
-	DefTiDBEnableFastReorg                         = false
-	DefTiDBDDLDiskQuota                            = 100 * 1024 * 1024 * 1024 // 100GB
-	DefExecutorConcurrency                         = 5
-	DefTiDBEnableGeneralPlanCache                  = false
-	DefTiDBGeneralPlanCacheSize                    = 100
-	DefTiDBEnableTiFlashReadForWriteStmt           = false
+	DefHostname                                     = "localhost"
+	DefIndexLookupConcurrency                       = ConcurrencyUnset
+	DefIndexLookupJoinConcurrency                   = ConcurrencyUnset
+	DefIndexSerialScanConcurrency                   = 1
+	DefIndexJoinBatchSize                           = 25000
+	DefIndexLookupSize                              = 20000
+	DefDistSQLScanConcurrency                       = 15
+	DefBuildStatsConcurrency                        = 4
+	DefAutoAnalyzeRatio                             = 0.5
+	DefAutoAnalyzeStartTime                         = "00:00 +0000"
+	DefAutoAnalyzeEndTime                           = "23:59 +0000"
+	DefAutoIncrementIncrement                       = 1
+	DefAutoIncrementOffset                          = 1
+	DefChecksumTableConcurrency                     = 4
+	DefSkipUTF8Check                                = false
+	DefSkipASCIICheck                               = false
+	DefOptAggPushDown                               = false
+	DefOptCartesianBCJ                              = 1
+	DefOptMPPOuterJoinFixedBuildSide                = false
+	DefOptWriteRowID                                = false
+	DefOptEnableCorrelationAdjustment               = true
+	DefOptLimitPushDownThreshold                    = 100
+	DefOptCorrelationThreshold                      = 0.9
+	DefOptCorrelationExpFactor                      = 1
+	DefOptCPUFactor                                 = 3.0
+	DefOptCopCPUFactor                              = 3.0
+	DefOptTiFlashConcurrencyFactor                  = 24.0
+	DefOptNetworkFactor                             = 1.0
+	DefOptScanFactor                                = 1.5
+	DefOptDescScanFactor                            = 3.0
+	DefOptSeekFactor                                = 20.0
+	DefOptMemoryFactor                              = 0.001
+	DefOptDiskFactor                                = 1.5
+	DefOptConcurrencyFactor                         = 3.0
+	DefOptForceInlineCTE                            = false
+	DefOptInSubqToJoinAndAgg                        = true
+	DefOptPreferRangeScan                           = false
+	DefBatchInsert                                  = false
+	DefBatchDelete                                  = false
+	DefBatchCommit                                  = false
+	DefCurretTS                                     = 0
+	DefInitChunkSize                                = 32
+	DefMinPagingSize                                = int(paging.MinPagingSize)
+	DefMaxPagingSize                                = int(paging.MaxPagingSize)
+	DefMaxChunkSize                                 = 1024
+	DefDMLBatchSize                                 = 0
+	DefMaxPreparedStmtCount                         = -1
+	DefWaitTimeout                                  = 28800
+	DefTiDBMemQuotaApplyCache                       = 32 << 20 // 32MB.
+	DefTiDBMemQuotaBindingCache                     = 64 << 20 // 64MB.
+	DefTiDBGeneralLog                               = false
+	DefTiDBPProfSQLCPU                              = 0
+	DefTiDBRetryLimit                               = 10
+	DefTiDBDisableTxnAutoRetry                      = true
+	DefTiDBConstraintCheckInPlace                   = false
+	DefTiDBHashJoinConcurrency                      = ConcurrencyUnset
+	DefTiDBProjectionConcurrency                    = ConcurrencyUnset
+	DefBroadcastJoinThresholdSize                   = 100 * 1024 * 1024
+	DefBroadcastJoinThresholdCount                  = 10 * 1024
+	DefTiDBOptimizerSelectivityLevel                = 0
+	DefTiDBOptimizerEnableNewOFGB                   = false
+	DefTiDBEnableOuterJoinReorder                   = false
+	DefTiDBEnableNAAJ                               = false
+	DefTiDBAllowBatchCop                            = 1
+	DefTiDBAllowMPPExecution                        = true
+	DefTiDBHashExchangeWithNewCollation             = true
+	DefTiDBEnforceMPPExecution                      = false
+	DefTiFlashMaxThreads                            = -1
+	DefTiDBMPPStoreFailTTL                          = "60s"
+	DefTiDBTxnMode                                  = ""
+	DefTiDBRowFormatV1                              = 1
+	DefTiDBRowFormatV2                              = 2
+	DefTiDBDDLReorgWorkerCount                      = 4
+	DefTiDBDDLReorgBatchSize                        = 256
+	DefTiDBDDLFlashbackConcurrency                  = 64
+	DefTiDBDDLErrorCountLimit                       = 512
+	DefTiDBMaxDeltaSchemaCount                      = 1024
+	DefTiDBPlacementMode                            = PlacementModeStrict
+	DefTiDBEnableAutoIncrementInGenerated           = false
+	DefTiDBHashAggPartialConcurrency                = ConcurrencyUnset
+	DefTiDBHashAggFinalConcurrency                  = ConcurrencyUnset
+	DefTiDBWindowConcurrency                        = ConcurrencyUnset
+	DefTiDBMergeJoinConcurrency                     = 1 // disable optimization by default
+	DefTiDBStreamAggConcurrency                     = 1
+	DefTiDBIndexMergeIntersectionConcurrency        = ConcurrencyUnset
+	DefTiDBForcePriority                            = mysql.NoPriority
+	DefEnableWindowFunction                         = true
+	DefEnablePipelinedWindowFunction                = true
+	DefEnableStrictDoubleTypeCheck                  = true
+	DefEnableVectorizedExpression                   = true
+	DefTiDBOptJoinReorderThreshold                  = 0
+	DefTiDBDDLSlowOprThreshold                      = 300
+	DefTiDBUseFastAnalyze                           = false
+	DefTiDBSkipIsolationLevelCheck                  = false
+	DefTiDBExpensiveQueryTimeThreshold              = 60 // 60s
+	DefTiDBScatterRegion                            = false
+	DefTiDBWaitSplitRegionFinish                    = true
+	DefWaitSplitRegionTimeout                       = 300 // 300s
+	DefTiDBEnableNoopFuncs                          = Off
+	DefTiDBEnableNoopVariables                      = true
+	DefTiDBAllowRemoveAutoInc                       = false
+	DefTiDBUsePlanBaselines                         = true
+	DefTiDBEvolvePlanBaselines                      = false
+	DefTiDBEvolvePlanTaskMaxTime                    = 600 // 600s
+	DefTiDBEvolvePlanTaskStartTime                  = "00:00 +0000"
+	DefTiDBEvolvePlanTaskEndTime                    = "23:59 +0000"
+	DefInnodbLockWaitTimeout                        = 50 // 50s
+	DefTiDBStoreLimit                               = 0
+	DefTiDBMetricSchemaStep                         = 60 // 60s
+	DefTiDBMetricSchemaRangeDuration                = 60 // 60s
+	DefTiDBFoundInPlanCache                         = false
+	DefTiDBFoundInBinding                           = false
+	DefTiDBEnableCollectExecutionInfo               = true
+	DefTiDBAllowAutoRandExplicitInsert              = false
+	DefTiDBEnableClusteredIndex                     = ClusteredIndexDefModeOn
+	DefTiDBRedactLog                                = false
+	DefTiDBRestrictedReadOnly                       = false
+	DefTiDBSuperReadOnly                            = false
+	DefTiDBShardAllocateStep                        = math.MaxInt64
+	DefTiDBEnableTelemetry                          = true
+	DefTiDBEnableParallelApply                      = false
+	DefTiDBEnableAmendPessimisticTxn                = false
+	DefTiDBPartitionPruneMode                       = "dynamic"
+	DefTiDBEnableRateLimitAction                    = false
+	DefTiDBEnableAsyncCommit                        = false
+	DefTiDBEnable1PC                                = false
+	DefTiDBGuaranteeLinearizability                 = true
+	DefTiDBAnalyzeVersion                           = 2
+	DefTiDBAutoAnalyzePartitionBatchSize            = 1
+	DefTiDBEnableIndexMergeJoin                     = false
+	DefTiDBTrackAggregateMemoryUsage                = true
+	DefTiDBEnableExchangePartition                  = true
+	DefCTEMaxRecursionDepth                         = 1000
+	DefTiDBTmpTableMaxSize                          = 64 << 20 // 64MB.
+	DefTiDBEnableLocalTxn                           = false
+	DefTiDBTSOClientBatchMaxWaitTime                = 0.0 // 0ms
+	DefTiDBEnableTSOFollowerProxy                   = false
+	DefTiDBEnableOrderedResultMode                  = false
+	DefTiDBEnablePseudoForOutdatedStats             = false
+	DefTiDBRegardNULLAsPoint                        = true
+	DefEnablePlacementCheck                         = true
+	DefTimestamp                                    = "0"
+	DefTimestampFloat                               = 0.0
+	DefTiDBEnableStmtSummary                        = true
+	DefTiDBStmtSummaryInternalQuery                 = false
+	DefTiDBStmtSummaryRefreshInterval               = 1800
+	DefTiDBStmtSummaryHistorySize                   = 24
+	DefTiDBStmtSummaryMaxStmtCount                  = 3000
+	DefTiDBStmtSummaryMaxSQLLength                  = 4096
+	DefTiDBCapturePlanBaseline                      = Off
+	DefTiDBEnableIndexMerge                         = true
+	DefEnableLegacyInstanceScope                    = true
+	DefTiDBTableCacheLease                          = 3 // 3s
+	DefTiDBPersistAnalyzeOptions                    = true
+	DefTiDBEnableColumnTracking                     = false
+	DefTiDBStatsLoadSyncWait                        = 100
+	DefTiDBStatsLoadPseudoTimeout                   = true
+	DefSysdateIsNow                                 = false
+	DefTiDBEnableMutationChecker                    = false
+	DefTiDBTxnAssertionLevel                        = AssertionOffStr
+	DefTiDBIgnorePreparedCacheCloseStmt             = false
+	DefTiDBBatchPendingTiFlashCount                 = 4000
+	DefRCReadCheckTS                                = false
+	DefTiDBRemoveOrderbyInSubquery                  = false
+	DefTiDBSkewDistinctAgg                          = false
+	DefTiDB3StageDistinctAgg                        = true
+	DefTiDBReadStaleness                            = 0
+	DefTiDBGCMaxWaitTime                            = 24 * 60 * 60
+	DefMaxAllowedPacket                      uint64 = 67108864
+	DefTiDBEnableBatchDML                           = false
+	DefTiDBMemQuotaQuery                            = 1073741824 // 1GB
+	DefTiDBStatsCacheMemQuota                       = 0
+	MaxTiDBStatsCacheMemQuota                       = 1024 * 1024 * 1024 * 1024 // 1TB
+	DefTiDBQueryLogMaxLen                           = 4096
+	DefRequireSecureTransport                       = false
+	DefTiDBCommitterConcurrency                     = 128
+	DefTiDBBatchDMLIgnoreError                      = false
+	DefTiDBMemQuotaAnalyze                          = -1
+	DefTiDBEnableAutoAnalyze                        = true
+	DefTiDBMemOOMAction                             = "CANCEL"
+	DefTiDBMaxAutoAnalyzeTime                       = 12 * 60 * 60
+	DefTiDBEnablePrepPlanCache                      = true
+	DefTiDBPrepPlanCacheSize                        = 100
+	DefTiDBEnablePrepPlanCacheMemoryMonitor         = true
+	DefTiDBPrepPlanCacheMemoryGuardRatio            = 0.1
+	DefTiDBEnableConcurrentDDL                      = concurrencyddl.TiDBEnableConcurrentDDL
+	DefTiDBSimplifiedMetrics                        = false
+	DefTiDBEnablePaging                             = true
+	DefTiFlashFineGrainedShuffleStreamCount         = 0
+	DefStreamCountWhenMaxThreadsNotSet              = 8
+	DefTiFlashFineGrainedShuffleBatchSize           = 8192
+	DefAdaptiveClosestReadThreshold                 = 4096
+	DefTiDBEnableAnalyzeSnapshot                    = false
+	DefTiDBGenerateBinaryPlan                       = true
+	DefEnableTiDBGCAwareMemoryTrack                 = true
+	DefTiDBDefaultStrMatchSelectivity               = 0.8
+	DefTiDBEnableTmpStorageOnOOM                    = true
+	DefTiDBEnableMDL                                = true
+	DefTiFlashFastScan                              = false
+	DefMemoryUsageAlarmRatio                        = 0.7
+	DefMemoryUsageAlarmKeepRecordNum                = 5
+	DefTiDBEnableFastReorg                          = false
+	DefTiDBDDLDiskQuota                             = 100 * 1024 * 1024 * 1024 // 100GB
+	DefExecutorConcurrency                          = 5
+	DefTiDBEnableGeneralPlanCache                   = false
+	DefTiDBGeneralPlanCacheSize                     = 100
+	DefTiDBEnableTiFlashReadForWriteStmt            = false
 	// MaxDDLReorgBatchSize is exported for testing.
 	MaxDDLReorgBatchSize                  int32  = 10240
 	MinDDLReorgBatchSize                  int32  = 32
diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go
index 69c9caf294e5e..cdd48a789b9b5 100644
--- a/sessionctx/variable/varsutil_test.go
+++ b/sessionctx/variable/varsutil_test.go
@@ -74,6 +74,7 @@ func TestNewSessionVars(t *testing.T) {
 	require.Equal(t, DefExecutorConcurrency, vars.HashAggPartialConcurrency())
 	require.Equal(t, DefExecutorConcurrency, vars.HashAggFinalConcurrency())
 	require.Equal(t, DefExecutorConcurrency, vars.WindowConcurrency())
+	require.Equal(t, DefExecutorConcurrency, vars.IndexMergeIntersectionConcurrency())
 	require.Equal(t, DefTiDBMergeJoinConcurrency, vars.MergeJoinConcurrency())
 	require.Equal(t, DefTiDBStreamAggConcurrency, vars.StreamAggConcurrency())
 	require.Equal(t, DefDistSQLScanConcurrency, vars.DistSQLScanConcurrency())

From 57136e9d8955f0245d17a131bc009524355f68c7 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Fri, 25 Nov 2022 13:38:16 +0800
Subject: [PATCH 16/33] del workerID

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 12288ccaf7c19..6570d04029aec 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -127,8 +127,7 @@ type IndexMergeReaderExecutor struct {
 type indexMergeTableTask struct {
 	lookupTableTask
 
-	// workerID and parTblIdx are only used in indexMergeProcessWorker.fetchLoopIntersection.
-	workerID  int
+	// parTblIdx are only used in indexMergeProcessWorker.fetchLoopIntersection.
 	parTblIdx int
 }
 
@@ -307,7 +306,6 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
 					batchSize:    e.maxChunkSize,
 					maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
 					maxChunkSize: e.maxChunkSize,
-					workerID:     workID,
 				}
 
 				if e.isCorColInPartialFilters[workID] {
@@ -425,7 +423,6 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context,
 					maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize,
 					maxChunkSize: e.maxChunkSize,
 					tableReader:  partialTableReader,
-					workerID:     workID,
 				}
 
 				if e.isCorColInPartialFilters[workID] {
@@ -514,7 +511,6 @@ type partialTableWorker struct {
 	maxChunkSize int
 	tableReader  Executor
 	partition    table.PhysicalTable // it indicates if this worker is accessing a particular partition table
-	workerID     int
 }
 
 func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask,
@@ -589,7 +585,6 @@ func (w *partialTableWorker) buildTableTask(handles []kv.Handle, retChk *chunk.C
 
 			partitionTable: w.partition,
 		},
-		workerID:  w.workerID,
 		parTblIdx: parTblIdx,
 	}
 
@@ -959,7 +954,6 @@ type partialIndexWorker struct {
 	maxBatchSize int
 	maxChunkSize int
 	partition    table.PhysicalTable // it indicates if this worker is accessing a particular partition table
-	workerID     int
 }
 
 func syncErr(resultCh chan<- *indexMergeTableTask, err error) {
@@ -1057,7 +1051,6 @@ func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.C
 
 			partitionTable: w.partition,
 		},
-		workerID:  w.workerID,
 		parTblIdx: parTblIdx,
 	}
 

From 4d4455e9f83784611cdba490f70374e49137e1c2 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 11:32:02 +0800
Subject: [PATCH 17/33] fis tidb_vars.go

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 sessionctx/variable/tidb_vars.go | 428 +++++++++++++++----------------
 1 file changed, 214 insertions(+), 214 deletions(-)

diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go
index c63f7c2b8e60a..6cad5f23b623d 100644
--- a/sessionctx/variable/tidb_vars.go
+++ b/sessionctx/variable/tidb_vars.go
@@ -874,213 +874,212 @@ const (
 
 // Default TiDB system variable values.
 const (
-	DefHostname                                     = "localhost"
-	DefIndexLookupConcurrency                       = ConcurrencyUnset
-	DefIndexLookupJoinConcurrency                   = ConcurrencyUnset
-	DefIndexSerialScanConcurrency                   = 1
-	DefIndexJoinBatchSize                           = 25000
-	DefIndexLookupSize                              = 20000
-	DefDistSQLScanConcurrency                       = 15
-	DefBuildStatsConcurrency                        = 4
-	DefAutoAnalyzeRatio                             = 0.5
-	DefAutoAnalyzeStartTime                         = "00:00 +0000"
-	DefAutoAnalyzeEndTime                           = "23:59 +0000"
-	DefAutoIncrementIncrement                       = 1
-	DefAutoIncrementOffset                          = 1
-	DefChecksumTableConcurrency                     = 4
-	DefSkipUTF8Check                                = false
-	DefSkipASCIICheck                               = false
-	DefOptAggPushDown                               = false
-	DefOptCartesianBCJ                              = 1
-	DefOptMPPOuterJoinFixedBuildSide                = false
-	DefOptWriteRowID                                = false
-	DefOptEnableCorrelationAdjustment               = true
-	DefOptLimitPushDownThreshold                    = 100
-	DefOptCorrelationThreshold                      = 0.9
-	DefOptCorrelationExpFactor                      = 1
-	DefOptCPUFactor                                 = 3.0
-	DefOptCopCPUFactor                              = 3.0
-	DefOptTiFlashConcurrencyFactor                  = 24.0
-	DefOptNetworkFactor                             = 1.0
-	DefOptScanFactor                                = 1.5
-	DefOptDescScanFactor                            = 3.0
-	DefOptSeekFactor                                = 20.0
-	DefOptMemoryFactor                              = 0.001
-	DefOptDiskFactor                                = 1.5
-	DefOptConcurrencyFactor                         = 3.0
-	DefOptForceInlineCTE                            = false
-	DefOptInSubqToJoinAndAgg                        = true
-	DefOptPreferRangeScan                           = false
-	DefBatchInsert                                  = false
-	DefBatchDelete                                  = false
-	DefBatchCommit                                  = false
-	DefCurretTS                                     = 0
-	DefInitChunkSize                                = 32
-	DefMinPagingSize                                = int(paging.MinPagingSize)
-	DefMaxPagingSize                                = int(paging.MaxPagingSize)
-	DefMaxChunkSize                                 = 1024
-	DefDMLBatchSize                                 = 0
-	DefMaxPreparedStmtCount                         = -1
-	DefWaitTimeout                                  = 28800
-	DefTiDBMemQuotaApplyCache                       = 32 << 20 // 32MB.
-	DefTiDBMemQuotaBindingCache                     = 64 << 20 // 64MB.
-	DefTiDBGeneralLog                               = false
-	DefTiDBPProfSQLCPU                              = 0
-	DefTiDBRetryLimit                               = 10
-	DefTiDBDisableTxnAutoRetry                      = true
-	DefTiDBConstraintCheckInPlace                   = false
-	DefTiDBHashJoinConcurrency                      = ConcurrencyUnset
-	DefTiDBProjectionConcurrency                    = ConcurrencyUnset
-	DefBroadcastJoinThresholdSize                   = 100 * 1024 * 1024
-	DefBroadcastJoinThresholdCount                  = 10 * 1024
-	DefTiDBOptimizerSelectivityLevel                = 0
-	DefTiDBOptimizerEnableNewOFGB                   = false
-	DefTiDBEnableOuterJoinReorder                   = false
-	DefTiDBEnableNAAJ                               = false
-	DefTiDBAllowBatchCop                            = 1
-	DefTiDBAllowMPPExecution                        = true
-	DefTiDBHashExchangeWithNewCollation             = true
-	DefTiDBEnforceMPPExecution                      = false
-	DefTiFlashMaxThreads                            = -1
-	DefTiDBMPPStoreFailTTL                          = "60s"
-	DefTiDBTxnMode                                  = ""
-	DefTiDBRowFormatV1                              = 1
-	DefTiDBRowFormatV2                              = 2
-	DefTiDBDDLReorgWorkerCount                      = 4
-	DefTiDBDDLReorgBatchSize                        = 256
-	DefTiDBDDLFlashbackConcurrency                  = 64
-	DefTiDBDDLErrorCountLimit                       = 512
-	DefTiDBMaxDeltaSchemaCount                      = 1024
-	DefTiDBPlacementMode                            = PlacementModeStrict
-	DefTiDBEnableAutoIncrementInGenerated           = false
-	DefTiDBHashAggPartialConcurrency                = ConcurrencyUnset
-	DefTiDBHashAggFinalConcurrency                  = ConcurrencyUnset
-	DefTiDBWindowConcurrency                        = ConcurrencyUnset
-	DefTiDBMergeJoinConcurrency                     = 1 // disable optimization by default
-	DefTiDBStreamAggConcurrency                     = 1
-	DefTiDBIndexMergeIntersectionConcurrency        = ConcurrencyUnset
-	DefTiDBForcePriority                            = mysql.NoPriority
-	DefEnableWindowFunction                         = true
-	DefEnablePipelinedWindowFunction                = true
-	DefEnableStrictDoubleTypeCheck                  = true
-	DefEnableVectorizedExpression                   = true
-	DefTiDBOptJoinReorderThreshold                  = 0
-	DefTiDBDDLSlowOprThreshold                      = 300
-	DefTiDBUseFastAnalyze                           = false
-	DefTiDBSkipIsolationLevelCheck                  = false
-	DefTiDBExpensiveQueryTimeThreshold              = 60 // 60s
-	DefTiDBScatterRegion                            = false
-	DefTiDBWaitSplitRegionFinish                    = true
-	DefWaitSplitRegionTimeout                       = 300 // 300s
-	DefTiDBEnableNoopFuncs                          = Off
-	DefTiDBEnableNoopVariables                      = true
-	DefTiDBAllowRemoveAutoInc                       = false
-	DefTiDBUsePlanBaselines                         = true
-	DefTiDBEvolvePlanBaselines                      = false
-	DefTiDBEvolvePlanTaskMaxTime                    = 600 // 600s
-	DefTiDBEvolvePlanTaskStartTime                  = "00:00 +0000"
-	DefTiDBEvolvePlanTaskEndTime                    = "23:59 +0000"
-	DefInnodbLockWaitTimeout                        = 50 // 50s
-	DefTiDBStoreLimit                               = 0
-	DefTiDBMetricSchemaStep                         = 60 // 60s
-	DefTiDBMetricSchemaRangeDuration                = 60 // 60s
-	DefTiDBFoundInPlanCache                         = false
-	DefTiDBFoundInBinding                           = false
-	DefTiDBEnableCollectExecutionInfo               = true
-	DefTiDBAllowAutoRandExplicitInsert              = false
-	DefTiDBEnableClusteredIndex                     = ClusteredIndexDefModeOn
-	DefTiDBRedactLog                                = false
-	DefTiDBRestrictedReadOnly                       = false
-	DefTiDBSuperReadOnly                            = false
-	DefTiDBShardAllocateStep                        = math.MaxInt64
-	DefTiDBEnableTelemetry                          = true
-	DefTiDBEnableParallelApply                      = false
-	DefTiDBEnableAmendPessimisticTxn                = false
-	DefTiDBPartitionPruneMode                       = "dynamic"
-	DefTiDBEnableRateLimitAction                    = false
-	DefTiDBEnableAsyncCommit                        = false
-	DefTiDBEnable1PC                                = false
-	DefTiDBGuaranteeLinearizability                 = true
-	DefTiDBAnalyzeVersion                           = 2
-	DefTiDBAutoAnalyzePartitionBatchSize            = 1
-	DefTiDBEnableIndexMergeJoin                     = false
-	DefTiDBTrackAggregateMemoryUsage                = true
-	DefTiDBEnableExchangePartition                  = true
-	DefCTEMaxRecursionDepth                         = 1000
-	DefTiDBTmpTableMaxSize                          = 64 << 20 // 64MB.
-	DefTiDBEnableLocalTxn                           = false
-	DefTiDBTSOClientBatchMaxWaitTime                = 0.0 // 0ms
-	DefTiDBEnableTSOFollowerProxy                   = false
-	DefTiDBEnableOrderedResultMode                  = false
-	DefTiDBEnablePseudoForOutdatedStats             = false
-	DefTiDBRegardNULLAsPoint                        = true
-	DefEnablePlacementCheck                         = true
-	DefTimestamp                                    = "0"
-	DefTimestampFloat                               = 0.0
-	DefTiDBEnableStmtSummary                        = true
-	DefTiDBStmtSummaryInternalQuery                 = false
-	DefTiDBStmtSummaryRefreshInterval               = 1800
-	DefTiDBStmtSummaryHistorySize                   = 24
-	DefTiDBStmtSummaryMaxStmtCount                  = 3000
-	DefTiDBStmtSummaryMaxSQLLength                  = 4096
-	DefTiDBCapturePlanBaseline                      = Off
-	DefTiDBEnableIndexMerge                         = true
-	DefEnableLegacyInstanceScope                    = true
-	DefTiDBTableCacheLease                          = 3 // 3s
-	DefTiDBPersistAnalyzeOptions                    = true
-	DefTiDBEnableColumnTracking                     = false
-	DefTiDBStatsLoadSyncWait                        = 100
-	DefTiDBStatsLoadPseudoTimeout                   = true
-	DefSysdateIsNow                                 = false
-	DefTiDBEnableMutationChecker                    = false
-	DefTiDBTxnAssertionLevel                        = AssertionOffStr
-	DefTiDBIgnorePreparedCacheCloseStmt             = false
-	DefTiDBBatchPendingTiFlashCount                 = 4000
-	DefRCReadCheckTS                                = false
-	DefTiDBRemoveOrderbyInSubquery                  = false
-	DefTiDBSkewDistinctAgg                          = false
-	DefTiDB3StageDistinctAgg                        = true
-	DefTiDBReadStaleness                            = 0
-	DefTiDBGCMaxWaitTime                            = 24 * 60 * 60
-	DefMaxAllowedPacket                      uint64 = 67108864
-	DefTiDBEnableBatchDML                           = false
-	DefTiDBMemQuotaQuery                            = 1073741824 // 1GB
-	DefTiDBStatsCacheMemQuota                       = 0
-	MaxTiDBStatsCacheMemQuota                       = 1024 * 1024 * 1024 * 1024 // 1TB
-	DefTiDBQueryLogMaxLen                           = 4096
-	DefRequireSecureTransport                       = false
-	DefTiDBCommitterConcurrency                     = 128
-	DefTiDBBatchDMLIgnoreError                      = false
-	DefTiDBMemQuotaAnalyze                          = -1
-	DefTiDBEnableAutoAnalyze                        = true
-	DefTiDBMemOOMAction                             = "CANCEL"
-	DefTiDBMaxAutoAnalyzeTime                       = 12 * 60 * 60
-	DefTiDBEnablePrepPlanCache                      = true
-	DefTiDBPrepPlanCacheSize                        = 100
-	DefTiDBEnablePrepPlanCacheMemoryMonitor         = true
-	DefTiDBPrepPlanCacheMemoryGuardRatio            = 0.1
-	DefTiDBEnableConcurrentDDL                      = concurrencyddl.TiDBEnableConcurrentDDL
-	DefTiDBSimplifiedMetrics                        = false
-	DefTiDBEnablePaging                             = true
-	DefTiFlashFineGrainedShuffleStreamCount         = 0
-	DefStreamCountWhenMaxThreadsNotSet              = 8
-	DefTiFlashFineGrainedShuffleBatchSize           = 8192
-	DefAdaptiveClosestReadThreshold                 = 4096
-	DefTiDBEnableAnalyzeSnapshot                    = false
-	DefTiDBGenerateBinaryPlan                       = true
-	DefEnableTiDBGCAwareMemoryTrack                 = true
-	DefTiDBDefaultStrMatchSelectivity               = 0.8
-	DefTiDBEnableTmpStorageOnOOM                    = true
-	DefTiDBEnableMDL                                = true
-	DefTiFlashFastScan                              = false
-	DefMemoryUsageAlarmRatio                        = 0.7
-	DefMemoryUsageAlarmKeepRecordNum                = 5
-	DefTiDBEnableFastReorg                          = true
-	DefTiDBDDLDiskQuota                             = 100 * 1024 * 1024 * 1024 // 100GB
-	DefExecutorConcurrency                          = 5
-	DefTiDBEnableGeneralPlanCache                   = false
-	DefTiDBGeneralPlanCacheSize                     = 100
-	DefTiDBEnableTiFlashReadForWriteStmt            = false
+	DefHostname                                    = "localhost"
+	DefIndexLookupConcurrency                      = ConcurrencyUnset
+	DefIndexLookupJoinConcurrency                  = ConcurrencyUnset
+	DefIndexSerialScanConcurrency                  = 1
+	DefIndexJoinBatchSize                          = 25000
+	DefIndexLookupSize                             = 20000
+	DefDistSQLScanConcurrency                      = 15
+	DefBuildStatsConcurrency                       = 4
+	DefAutoAnalyzeRatio                            = 0.5
+	DefAutoAnalyzeStartTime                        = "00:00 +0000"
+	DefAutoAnalyzeEndTime                          = "23:59 +0000"
+	DefAutoIncrementIncrement                      = 1
+	DefAutoIncrementOffset                         = 1
+	DefChecksumTableConcurrency                    = 4
+	DefSkipUTF8Check                               = false
+	DefSkipASCIICheck                              = false
+	DefOptAggPushDown                              = false
+	DefOptCartesianBCJ                             = 1
+	DefOptMPPOuterJoinFixedBuildSide               = false
+	DefOptWriteRowID                               = false
+	DefOptEnableCorrelationAdjustment              = true
+	DefOptLimitPushDownThreshold                   = 100
+	DefOptCorrelationThreshold                     = 0.9
+	DefOptCorrelationExpFactor                     = 1
+	DefOptCPUFactor                                = 3.0
+	DefOptCopCPUFactor                             = 3.0
+	DefOptTiFlashConcurrencyFactor                 = 24.0
+	DefOptNetworkFactor                            = 1.0
+	DefOptScanFactor                               = 1.5
+	DefOptDescScanFactor                           = 3.0
+	DefOptSeekFactor                               = 20.0
+	DefOptMemoryFactor                             = 0.001
+	DefOptDiskFactor                               = 1.5
+	DefOptConcurrencyFactor                        = 3.0
+	DefOptForceInlineCTE                           = false
+	DefOptInSubqToJoinAndAgg                       = true
+	DefOptPreferRangeScan                          = false
+	DefBatchInsert                                 = false
+	DefBatchDelete                                 = false
+	DefBatchCommit                                 = false
+	DefCurretTS                                    = 0
+	DefInitChunkSize                               = 32
+	DefMinPagingSize                               = int(paging.MinPagingSize)
+	DefMaxPagingSize                               = int(paging.MaxPagingSize)
+	DefMaxChunkSize                                = 1024
+	DefDMLBatchSize                                = 0
+	DefMaxPreparedStmtCount                        = -1
+	DefWaitTimeout                                 = 28800
+	DefTiDBMemQuotaApplyCache                      = 32 << 20 // 32MB.
+	DefTiDBMemQuotaBindingCache                    = 64 << 20 // 64MB.
+	DefTiDBGeneralLog                              = false
+	DefTiDBPProfSQLCPU                             = 0
+	DefTiDBRetryLimit                              = 10
+	DefTiDBDisableTxnAutoRetry                     = true
+	DefTiDBConstraintCheckInPlace                  = false
+	DefTiDBHashJoinConcurrency                     = ConcurrencyUnset
+	DefTiDBProjectionConcurrency                   = ConcurrencyUnset
+	DefBroadcastJoinThresholdSize                  = 100 * 1024 * 1024
+	DefBroadcastJoinThresholdCount                 = 10 * 1024
+	DefTiDBOptimizerSelectivityLevel               = 0
+	DefTiDBOptimizerEnableNewOFGB                  = false
+	DefTiDBEnableOuterJoinReorder                  = false
+	DefTiDBEnableNAAJ                              = false
+	DefTiDBAllowBatchCop                           = 1
+	DefTiDBAllowMPPExecution                       = true
+	DefTiDBHashExchangeWithNewCollation            = true
+	DefTiDBEnforceMPPExecution                     = false
+	DefTiFlashMaxThreads                           = -1
+	DefTiDBMPPStoreFailTTL                         = "60s"
+	DefTiDBTxnMode                                 = ""
+	DefTiDBRowFormatV1                             = 1
+	DefTiDBRowFormatV2                             = 2
+	DefTiDBDDLReorgWorkerCount                     = 4
+	DefTiDBDDLReorgBatchSize                       = 256
+	DefTiDBDDLFlashbackConcurrency                 = 64
+	DefTiDBDDLErrorCountLimit                      = 512
+	DefTiDBMaxDeltaSchemaCount                     = 1024
+	DefTiDBPlacementMode                           = PlacementModeStrict
+	DefTiDBEnableAutoIncrementInGenerated          = false
+	DefTiDBHashAggPartialConcurrency               = ConcurrencyUnset
+	DefTiDBHashAggFinalConcurrency                 = ConcurrencyUnset
+	DefTiDBWindowConcurrency                       = ConcurrencyUnset
+	DefTiDBMergeJoinConcurrency                    = 1 // disable optimization by default
+	DefTiDBStreamAggConcurrency                    = 1
+	DefTiDBForcePriority                           = mysql.NoPriority
+	DefEnableWindowFunction                        = true
+	DefEnablePipelinedWindowFunction               = true
+	DefEnableStrictDoubleTypeCheck                 = true
+	DefEnableVectorizedExpression                  = true
+	DefTiDBOptJoinReorderThreshold                 = 0
+	DefTiDBDDLSlowOprThreshold                     = 300
+	DefTiDBUseFastAnalyze                          = false
+	DefTiDBSkipIsolationLevelCheck                 = false
+	DefTiDBExpensiveQueryTimeThreshold             = 60 // 60s
+	DefTiDBScatterRegion                           = false
+	DefTiDBWaitSplitRegionFinish                   = true
+	DefWaitSplitRegionTimeout                      = 300 // 300s
+	DefTiDBEnableNoopFuncs                         = Off
+	DefTiDBEnableNoopVariables                     = true
+	DefTiDBAllowRemoveAutoInc                      = false
+	DefTiDBUsePlanBaselines                        = true
+	DefTiDBEvolvePlanBaselines                     = false
+	DefTiDBEvolvePlanTaskMaxTime                   = 600 // 600s
+	DefTiDBEvolvePlanTaskStartTime                 = "00:00 +0000"
+	DefTiDBEvolvePlanTaskEndTime                   = "23:59 +0000"
+	DefInnodbLockWaitTimeout                       = 50 // 50s
+	DefTiDBStoreLimit                              = 0
+	DefTiDBMetricSchemaStep                        = 60 // 60s
+	DefTiDBMetricSchemaRangeDuration               = 60 // 60s
+	DefTiDBFoundInPlanCache                        = false
+	DefTiDBFoundInBinding                          = false
+	DefTiDBEnableCollectExecutionInfo              = true
+	DefTiDBAllowAutoRandExplicitInsert             = false
+	DefTiDBEnableClusteredIndex                    = ClusteredIndexDefModeOn
+	DefTiDBRedactLog                               = false
+	DefTiDBRestrictedReadOnly                      = false
+	DefTiDBSuperReadOnly                           = false
+	DefTiDBShardAllocateStep                       = math.MaxInt64
+	DefTiDBEnableTelemetry                         = true
+	DefTiDBEnableParallelApply                     = false
+	DefTiDBEnableAmendPessimisticTxn               = false
+	DefTiDBPartitionPruneMode                      = "dynamic"
+	DefTiDBEnableRateLimitAction                   = false
+	DefTiDBEnableAsyncCommit                       = false
+	DefTiDBEnable1PC                               = false
+	DefTiDBGuaranteeLinearizability                = true
+	DefTiDBAnalyzeVersion                          = 2
+	DefTiDBAutoAnalyzePartitionBatchSize           = 1
+	DefTiDBEnableIndexMergeJoin                    = false
+	DefTiDBTrackAggregateMemoryUsage               = true
+	DefTiDBEnableExchangePartition                 = true
+	DefCTEMaxRecursionDepth                        = 1000
+	DefTiDBTmpTableMaxSize                         = 64 << 20 // 64MB.
+	DefTiDBEnableLocalTxn                          = false
+	DefTiDBTSOClientBatchMaxWaitTime               = 0.0 // 0ms
+	DefTiDBEnableTSOFollowerProxy                  = false
+	DefTiDBEnableOrderedResultMode                 = false
+	DefTiDBEnablePseudoForOutdatedStats            = false
+	DefTiDBRegardNULLAsPoint                       = true
+	DefEnablePlacementCheck                        = true
+	DefTimestamp                                   = "0"
+	DefTimestampFloat                              = 0.0
+	DefTiDBEnableStmtSummary                       = true
+	DefTiDBStmtSummaryInternalQuery                = false
+	DefTiDBStmtSummaryRefreshInterval              = 1800
+	DefTiDBStmtSummaryHistorySize                  = 24
+	DefTiDBStmtSummaryMaxStmtCount                 = 3000
+	DefTiDBStmtSummaryMaxSQLLength                 = 4096
+	DefTiDBCapturePlanBaseline                     = Off
+	DefTiDBEnableIndexMerge                        = true
+	DefEnableLegacyInstanceScope                   = true
+	DefTiDBTableCacheLease                         = 3 // 3s
+	DefTiDBPersistAnalyzeOptions                   = true
+	DefTiDBEnableColumnTracking                    = false
+	DefTiDBStatsLoadSyncWait                       = 100
+	DefTiDBStatsLoadPseudoTimeout                  = true
+	DefSysdateIsNow                                = false
+	DefTiDBEnableMutationChecker                   = false
+	DefTiDBTxnAssertionLevel                       = AssertionOffStr
+	DefTiDBIgnorePreparedCacheCloseStmt            = false
+	DefTiDBBatchPendingTiFlashCount                = 4000
+	DefRCReadCheckTS                               = false
+	DefTiDBRemoveOrderbyInSubquery                 = false
+	DefTiDBSkewDistinctAgg                         = false
+	DefTiDB3StageDistinctAgg                       = true
+	DefTiDBReadStaleness                           = 0
+	DefTiDBGCMaxWaitTime                           = 24 * 60 * 60
+	DefMaxAllowedPacket                     uint64 = 67108864
+	DefTiDBEnableBatchDML                          = false
+	DefTiDBMemQuotaQuery                           = 1073741824 // 1GB
+	DefTiDBStatsCacheMemQuota                      = 0
+	MaxTiDBStatsCacheMemQuota                      = 1024 * 1024 * 1024 * 1024 // 1TB
+	DefTiDBQueryLogMaxLen                          = 4096
+	DefRequireSecureTransport                      = false
+	DefTiDBCommitterConcurrency                    = 128
+	DefTiDBBatchDMLIgnoreError                     = false
+	DefTiDBMemQuotaAnalyze                         = -1
+	DefTiDBEnableAutoAnalyze                       = true
+	DefTiDBMemOOMAction                            = "CANCEL"
+	DefTiDBMaxAutoAnalyzeTime                      = 12 * 60 * 60
+	DefTiDBEnablePrepPlanCache                     = true
+	DefTiDBPrepPlanCacheSize                       = 100
+	DefTiDBEnablePrepPlanCacheMemoryMonitor        = true
+	DefTiDBPrepPlanCacheMemoryGuardRatio           = 0.1
+	DefTiDBEnableConcurrentDDL                     = concurrencyddl.TiDBEnableConcurrentDDL
+	DefTiDBSimplifiedMetrics                       = false
+	DefTiDBEnablePaging                            = true
+	DefTiFlashFineGrainedShuffleStreamCount        = 0
+	DefStreamCountWhenMaxThreadsNotSet             = 8
+	DefTiFlashFineGrainedShuffleBatchSize          = 8192
+	DefAdaptiveClosestReadThreshold                = 4096
+	DefTiDBEnableAnalyzeSnapshot                   = false
+	DefTiDBGenerateBinaryPlan                      = true
+	DefEnableTiDBGCAwareMemoryTrack                = true
+	DefTiDBDefaultStrMatchSelectivity              = 0.8
+	DefTiDBEnableTmpStorageOnOOM                   = true
+	DefTiDBEnableMDL                               = true
+	DefTiFlashFastScan                             = false
+	DefMemoryUsageAlarmRatio                       = 0.7
+	DefMemoryUsageAlarmKeepRecordNum               = 5
+	DefTiDBEnableFastReorg                         = true
+	DefTiDBDDLDiskQuota                            = 100 * 1024 * 1024 * 1024 // 100GB
+	DefExecutorConcurrency                         = 5
+	DefTiDBEnableGeneralPlanCache                  = false
+	DefTiDBGeneralPlanCacheSize                    = 100
+	DefTiDBEnableTiFlashReadForWriteStmt           = false
 	// MaxDDLReorgBatchSize is exported for testing.
 	MaxDDLReorgBatchSize                  int32  = 10240
 	MinDDLReorgBatchSize                  int32  = 32
@@ -1095,13 +1094,14 @@ const (
 	DefTiDBServerMemoryLimitGCTrigger            = 0.7
 	DefTiDBEnableGOGCTuner                       = true
 	// DefTiDBGOGCTunerThreshold is to limit TiDBGOGCTunerThreshold.
-	DefTiDBGOGCTunerThreshold        float64 = 0.6
-	DefTiDBOptPrefixIndexSingleScan          = true
-	DefTiDBExternalTS                        = 0
-	DefTiDBEnableExternalTSRead              = false
-	DefTiDBEnableReusechunk                  = true
-	DefTiDBUseAlloc                          = false
-	DefTiDBEnablePlanReplayerCapture         = false
+	DefTiDBGOGCTunerThreshold                float64 = 0.6
+	DefTiDBOptPrefixIndexSingleScan                  = true
+	DefTiDBExternalTS                                = 0
+	DefTiDBEnableExternalTSRead                      = false
+	DefTiDBEnableReusechunk                          = true
+	DefTiDBUseAlloc                                  = false
+	DefTiDBEnablePlanReplayerCapture                 = false
+	DefTiDBIndexMergeIntersectionConcurrency         = ConcurrencyUnset
 )
 
 // Process global variables.

From 7e6559a2aa75cccbd7cf591ef9062998da9e662c Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 12:25:17 +0800
Subject: [PATCH 18/33] refine Consume

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 6570d04029aec..f4d4f18882ba2 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -806,18 +806,18 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 		if hMap, ok = w.handleMapsPerWorker[task.parTblIdx]; !ok {
 			panic(fmt.Sprintf("cannot find parTblIdx(%d) for worker(id: %d)", task.parTblIdx, w.workerID))
 		}
-		oriCnt := hMap.Len()
+		var deltaRows int64
 		for _, h := range task.handles {
 			if cntPtr, ok := hMap.Get(h); ok {
 				(*cntPtr.(*int))++
 			} else {
 				cnt := 1
 				hMap.Set(h, &cnt)
+				deltaRows++
 			}
 		}
-		newCnt := hMap.Len()
-		if newCnt > oriCnt {
-			w.memTracker.Consume(int64(newCnt-oriCnt) * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
+		if deltaRows > 0 {
+			w.memTracker.Consume(deltaRows * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
 		}
 	}
 	w.wg.Done()
@@ -886,10 +886,15 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	for task := range fetchCh {
 		workers[task.parTblIdx/partCntPerWorker].workerCh <- task
 	}
-	for _, w := range workers {
-		close(w.workerCh)
+	for _, processWorker := range workers {
+		close(processWorker.workerCh)
 	}
 	wg.Wait()
+	defer func() {
+		for _, hMap := range handleMaps {
+			w.indexMerge.memTracker.Consume(-(int64(hMap.Len()) * (8 + int64(unsafe.Sizeof(reflect.Pointer)))))
+		}
+	}()
 
 	intersected := make([][]kv.Handle, partCnt)
 	for parTblIdx, hMap := range handleMaps {

From a8528735922e0845c72da431f9099cb7dc6180e3 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 13:31:30 +0800
Subject: [PATCH 19/33] use memAwareMap

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 38 ++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index f4d4f18882ba2..fb2b70be3b296 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -792,32 +792,36 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 type intersectionProcessWorker struct {
 	workerID int
 	// key: parTblIdx, val: HandleMap
-	handleMapsPerWorker map[int]*kv.HandleMap
+	handleMapsPerWorker map[int]*kv.MemAwareHandleMap[*int]
 	workerCh            chan *indexMergeTableTask
 	indexMerge          *IndexMergeReaderExecutor
 	wg                  *sync.WaitGroup
 	memTracker          *memory.Tracker
+	memUsage            int64
 }
 
 func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 	for task := range w.workerCh {
 		var ok bool
-		var hMap *kv.HandleMap
+		var hMap *kv.MemAwareHandleMap[*int]
 		if hMap, ok = w.handleMapsPerWorker[task.parTblIdx]; !ok {
 			panic(fmt.Sprintf("cannot find parTblIdx(%d) for worker(id: %d)", task.parTblIdx, w.workerID))
 		}
-		var deltaRows int64
+		var mapDelta int64
+		var rowDelta int64
 		for _, h := range task.handles {
 			if cntPtr, ok := hMap.Get(h); ok {
-				(*cntPtr.(*int))++
+				(*cntPtr)++
 			} else {
 				cnt := 1
-				hMap.Set(h, &cnt)
-				deltaRows++
+				mapDelta += hMap.Set(h, &cnt)
+				rowDelta += 1
 			}
 		}
-		if deltaRows > 0 {
-			w.memTracker.Consume(deltaRows * (8 + int64(unsafe.Sizeof(reflect.Pointer))))
+		if rowDelta > 0 {
+			memDelta := mapDelta + (rowDelta * int64(unsafe.Sizeof(reflect.Pointer)))
+			w.memUsage += memDelta
+			w.memTracker.Consume(mapDelta)
 		}
 	}
 	w.wg.Done()
@@ -861,13 +865,13 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			panic(fmt.Sprintf("unexpected workerCnt, expect %d, got %d", con, workerCnt))
 		}
 	})
-	handleMaps := make([]*kv.HandleMap, 0, partCnt)
+	handleMaps := make([]*kv.MemAwareHandleMap[*int], 0, partCnt)
 	workers := make([]*intersectionProcessWorker, 0, workerCnt)
 	wg := sync.WaitGroup{}
 	for i := 0; i < workerCnt; i++ {
-		handleMapsPerWorker := make(map[int]*kv.HandleMap, partCntPerWorker)
+		handleMapsPerWorker := make(map[int]*kv.MemAwareHandleMap[*int], partCntPerWorker)
 		for j := 0; j < partCntPerWorker; j++ {
-			hMap := kv.NewHandleMap()
+			hMap := kv.NewMemAwareHandleMap[*int]()
 			handleMaps = append(handleMaps, hMap)
 			parTblIdx := i*partCntPerWorker + j
 			handleMapsPerWorker[parTblIdx] = hMap
@@ -890,11 +894,13 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		close(processWorker.workerCh)
 	}
 	wg.Wait()
-	defer func() {
-		for _, hMap := range handleMaps {
-			w.indexMerge.memTracker.Consume(-(int64(hMap.Len()) * (8 + int64(unsafe.Sizeof(reflect.Pointer)))))
-		}
-	}()
+	var allMemUsage int64
+	for _, processWorker := range workers {
+		allMemUsage += processWorker.memUsage
+		defer func() {
+			w.indexMerge.memTracker.Consume(-(allMemUsage))
+		}()
+	}
 
 	intersected := make([][]kv.Handle, partCnt)
 	for parTblIdx, hMap := range handleMaps {

From e1f74f69e84937e4e2e1d5abd6d3d5b787f980df Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 16:27:54 +0800
Subject: [PATCH 20/33] fix mapDelta

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index fb2b70be3b296..dca7a1dd0dbf6 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -814,7 +814,7 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 				(*cntPtr)++
 			} else {
 				cnt := 1
-				mapDelta += hMap.Set(h, &cnt)
+				mapDelta += hMap.Set(h, &cnt) + int64(h.MemUsage())
 				rowDelta += 1
 			}
 		}

From 87369c638bc54f8d405ea2d1825e1f80f9816a99 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 16:35:31 +0800
Subject: [PATCH 21/33] fix ExtraMemSize

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index dca7a1dd0dbf6..55b6a567d1d3c 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -814,7 +814,7 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 				(*cntPtr)++
 			} else {
 				cnt := 1
-				mapDelta += hMap.Set(h, &cnt) + int64(h.MemUsage())
+				mapDelta += hMap.Set(h, &cnt) + int64(h.ExtraMemSize())
 				rowDelta += 1
 			}
 		}

From 49b9e5ff2878c01587bba9f3b4e3cb6ce340c032 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 16:43:55 +0800
Subject: [PATCH 22/33] fix defer

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 55b6a567d1d3c..8990fffb549eb 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -897,10 +897,10 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	var allMemUsage int64
 	for _, processWorker := range workers {
 		allMemUsage += processWorker.memUsage
-		defer func() {
-			w.indexMerge.memTracker.Consume(-(allMemUsage))
-		}()
 	}
+	defer func() {
+		w.indexMerge.memTracker.Consume(-(allMemUsage))
+	}()
 
 	intersected := make([][]kv.Handle, partCnt)
 	for parTblIdx, hMap := range handleMaps {

From e750ce9fbbd0538396e6fe2d829383fb03f1485b Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 17:03:25 +0800
Subject: [PATCH 23/33] fix

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 sessionctx/variable/session.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go
index 2331adb8d0875..8c3043bf0e924 100644
--- a/sessionctx/variable/session.go
+++ b/sessionctx/variable/session.go
@@ -2457,6 +2457,7 @@ func (c *Concurrency) SetStreamAggConcurrency(n int) {
 	c.streamAggConcurrency = n
 }
 
+// SetIndexMergeIntersectionConcurrency set the number of concurrent intersection process worker.
 func (c *Concurrency) SetIndexMergeIntersectionConcurrency(n int) {
 	c.indexMergeIntersectionConcurrency = n
 }
@@ -2543,6 +2544,7 @@ func (c *Concurrency) StreamAggConcurrency() int {
 	return c.ExecutorConcurrency
 }
 
+// IndexMergeIntersectionConcurrency return the number of concurrent process worker.
 func (c *Concurrency) IndexMergeIntersectionConcurrency() int {
 	if c.indexMergeIntersectionConcurrency != ConcurrencyUnset {
 		return c.indexMergeIntersectionConcurrency

From 5136dac82f7a42aadd8f239dcc2988302d7aa39a Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Mon, 28 Nov 2022 23:26:56 +0800
Subject: [PATCH 24/33] split task; recoverfn

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 122 +++++++++++++++++++++++++--------
 1 file changed, 92 insertions(+), 30 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 8990fffb549eb..8e84045776f7a 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -269,7 +269,7 @@ func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Cont
 					idxMergeProcessWorker.fetchLoopUnion(ctx, fetch, workCh, e.resultCh, e.finished)
 				}
 			},
-			idxMergeProcessWorker.handleLoopFetcherPanic(ctx, e.resultCh),
+			idxMergeProcessWorker.handleLoopFetcherPanic(ctx, e.resultCh, "IndexMergeProcessWorker", nil),
 		)
 		e.processWokerWg.Done()
 	}()
@@ -790,17 +790,35 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 }
 
 type intersectionProcessWorker struct {
-	workerID int
 	// key: parTblIdx, val: HandleMap
 	handleMapsPerWorker map[int]*kv.MemAwareHandleMap[*int]
+	workerID            int
 	workerCh            chan *indexMergeTableTask
 	indexMerge          *IndexMergeReaderExecutor
-	wg                  *sync.WaitGroup
 	memTracker          *memory.Tracker
-	memUsage            int64
+
+	// When rowDelta == memConsumeBatchSize, Consume(memUsageDelta)
+	memUsageDelta       int64
+	rowDelta            int64
+	memConsumeBatchSize int
+
+	// For Consume(-totalMemUsage)
+	totalMemUsage int64
+}
+
+func (w *intersectionProcessWorker) consumeMemDelta() {
+	w.memTracker.Consume(w.memUsageDelta)
+	w.memUsageDelta = 0
+	w.rowDelta = 0
 }
 
 func (w *intersectionProcessWorker) doIntersectionPerPartition() {
+	defer func() {
+		if w.rowDelta > 0 {
+			w.consumeMemDelta()
+		}
+	}()
+
 	for task := range w.workerCh {
 		var ok bool
 		var hMap *kv.MemAwareHandleMap[*int]
@@ -818,13 +836,19 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 				rowDelta += 1
 			}
 		}
+
+		logutil.BgLogger().Debug("intersectionProcessWorker handle tasks", zap.Int("workerID", w.workerID),
+			zap.Int("task.handles", len(task.handles)), zap.Int64("rowDelta", rowDelta))
 		if rowDelta > 0 {
-			memDelta := mapDelta + (rowDelta * int64(unsafe.Sizeof(reflect.Pointer)))
-			w.memUsage += memDelta
-			w.memTracker.Consume(mapDelta)
+			w.rowDelta += rowDelta
+			w.memUsageDelta += mapDelta + (rowDelta * int64(unsafe.Sizeof(reflect.Int64)))
+			w.totalMemUsage += w.memUsageDelta
+		}
+		if w.rowDelta >= int64(w.memConsumeBatchSize) {
+			w.consumeMemDelta()
 		}
+		failpoint.Inject("testIndexMergeIntersectionWorkerPanic", nil)
 	}
-	w.wg.Done()
 }
 
 func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
@@ -841,13 +865,15 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		}()
 	}
 
-	partCnt := 1
-	workerCnt := 1
-	partCntPerWorker := 1
 	// One goroutine may handle one or multiple partitions.
 	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
 	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
-	maxChannelSize := 1024
+	partCnt := 1
+	workerCnt := 1
+	partCntPerWorker := 1
+
+	const maxChannelSize int = 1024
+	batchSize := w.indexMerge.ctx.GetSessionVars().IndexLookupSize
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
 		workerCnt = partCnt
@@ -868,6 +894,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	handleMaps := make([]*kv.MemAwareHandleMap[*int], 0, partCnt)
 	workers := make([]*intersectionProcessWorker, 0, workerCnt)
 	wg := sync.WaitGroup{}
+	errCh := make(chan bool, workerCnt)
 	for i := 0; i < workerCnt; i++ {
 		handleMapsPerWorker := make(map[int]*kv.MemAwareHandleMap[*int], partCntPerWorker)
 		for j := 0; j < partCntPerWorker; j++ {
@@ -876,32 +903,52 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			parTblIdx := i*partCntPerWorker + j
 			handleMapsPerWorker[parTblIdx] = hMap
 		}
-		workers = append(workers, &intersectionProcessWorker{
+		worker := &intersectionProcessWorker{
 			workerID:            i,
 			handleMapsPerWorker: handleMapsPerWorker,
 			workerCh:            make(chan *indexMergeTableTask, maxChannelSize),
 			indexMerge:          w.indexMerge,
-			wg:                  &wg,
 			memTracker:          w.indexMerge.memTracker,
-		})
-		go workers[i].doIntersectionPerPartition()
+			memConsumeBatchSize: batchSize,
+		}
 		wg.Add(1)
+		go func() {
+			defer trace.StartRegion(ctx, "IndexMergeIntersectionProcessWorker").End()
+			defer wg.Done()
+			util.WithRecovery(
+				func() {
+					worker.doIntersectionPerPartition()
+				},
+				w.handleLoopFetcherPanic(ctx, resultCh, "IndexMergeIntersectionProcessWorker", errCh),
+			)
+		}()
+		workers = append(workers, worker)
 	}
+	var processWorkerErr bool
 	for task := range fetchCh {
-		workers[task.parTblIdx/partCntPerWorker].workerCh <- task
+		select {
+		case workers[task.parTblIdx/partCntPerWorker].workerCh <- task:
+		case processWorkerErr = <-errCh:
+			break
+		}
 	}
 	for _, processWorker := range workers {
 		close(processWorker.workerCh)
 	}
 	wg.Wait()
-	var allMemUsage int64
+	if processWorkerErr {
+		return
+	}
+
+	var totalMemUsage int64
 	for _, processWorker := range workers {
-		allMemUsage += processWorker.memUsage
+		totalMemUsage += processWorker.totalMemUsage
 	}
 	defer func() {
-		w.indexMerge.memTracker.Consume(-(allMemUsage))
+		w.indexMerge.memTracker.Consume(-(totalMemUsage))
 	}()
 
+	// We assume the result of intersection is small, so no need to track memory.
 	intersected := make([][]kv.Handle, partCnt)
 	for parTblIdx, hMap := range handleMaps {
 		hMap.Range(func(h kv.Handle, val interface{}) bool {
@@ -917,14 +964,26 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		if len(intersected[parTblIdx]) == 0 {
 			continue
 		}
-		tasks = append(tasks, &indexMergeTableTask{
-			lookupTableTask: lookupTableTask{
-				handles: intersected[parTblIdx],
-				doneCh:  make(chan error, 1),
-			},
-		})
-		if w.indexMerge.partitionTableMode {
-			tasks[len(tasks)-1].partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
+		// Split intersected[parTblIdx] to avoid task is too large.
+		for len(intersected[parTblIdx]) > 0 {
+			length := batchSize
+			if length > len(intersected[parTblIdx]) {
+				length = len(intersected[parTblIdx])
+			}
+			s := intersected[parTblIdx][:length]
+			intersected[parTblIdx] = intersected[parTblIdx][length:]
+			task := &indexMergeTableTask{
+				lookupTableTask: lookupTableTask{
+					handles: s,
+					doneCh:  make(chan error, 1),
+				},
+			}
+			if w.indexMerge.partitionTableMode {
+				task.partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
+			}
+			tasks = append(tasks, task)
+			logutil.BgLogger().Debug("intersectionProcessWorker build tasks",
+				zap.Int("parTblIdx", parTblIdx), zap.Int("task.handles", len(task.handles)))
 		}
 	}
 	for _, task := range tasks {
@@ -939,13 +998,16 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	}
 }
 
-func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask) func(r interface{}) {
+func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask, worker string, extraCh chan bool) func(r interface{}) {
 	return func(r interface{}) {
 		if r == nil {
 			return
 		}
+		if extraCh != nil {
+			extraCh <- true
+		}
 
-		err4Panic := errors.Errorf("panic in IndexMergeReaderExecutor indexMergeTableWorker: %v", r)
+		err4Panic := errors.Errorf("panic in IndexMergeReaderExecutor %s: %v", worker, r)
 		logutil.Logger(ctx).Error(err4Panic.Error())
 		doneCh := make(chan error, 1)
 		doneCh <- err4Panic

From d25a88a3ba3164a758a2ab33bb3a8f6fa4d2385f Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 13:09:49 +0800
Subject: [PATCH 25/33] add comment

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 8e84045776f7a..c0c965615a876 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -851,6 +851,9 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 	}
 }
 
+// For each partition(dynamic mode), a map is used to do intersection. Key of the map is handle, and value is the number of times it occurs.
+// If the value of handle equals the number of partial paths, it should be sent to final_table_scan_worker.
+// To avoid too many goroutines, each intersectionProcessWorker can handle multiple partitions.
 func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fetchCh <-chan *indexMergeTableTask,
 	workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
 	defer func() {

From f5f70ac0146726101c2d769fe2d7277d94509bef Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 14:34:55 +0800
Subject: [PATCH 26/33] fix comment

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 134 +++++++++++++++------------------
 1 file changed, 61 insertions(+), 73 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index c0c965615a876..21645f3060b61 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -791,16 +791,17 @@ func (w *indexMergeProcessWorker) fetchLoopUnion(ctx context.Context, fetchCh <-
 
 type intersectionProcessWorker struct {
 	// key: parTblIdx, val: HandleMap
+	// Value of MemAwareHandleMap is *int to avoid extra Get().
 	handleMapsPerWorker map[int]*kv.MemAwareHandleMap[*int]
 	workerID            int
 	workerCh            chan *indexMergeTableTask
 	indexMerge          *IndexMergeReaderExecutor
 	memTracker          *memory.Tracker
+	batchSize           int
 
 	// When rowDelta == memConsumeBatchSize, Consume(memUsageDelta)
-	memUsageDelta       int64
-	rowDelta            int64
-	memConsumeBatchSize int
+	memUsageDelta int64
+	rowDelta      int64
 
 	// For Consume(-totalMemUsage)
 	totalMemUsage int64
@@ -812,7 +813,7 @@ func (w *intersectionProcessWorker) consumeMemDelta() {
 	w.rowDelta = 0
 }
 
-func (w *intersectionProcessWorker) doIntersectionPerPartition() {
+func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Context, workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
 	defer func() {
 		if w.rowDelta > 0 {
 			w.consumeMemDelta()
@@ -823,7 +824,8 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 		var ok bool
 		var hMap *kv.MemAwareHandleMap[*int]
 		if hMap, ok = w.handleMapsPerWorker[task.parTblIdx]; !ok {
-			panic(fmt.Sprintf("cannot find parTblIdx(%d) for worker(id: %d)", task.parTblIdx, w.workerID))
+			hMap = kv.NewMemAwareHandleMap[*int]()
+			w.handleMapsPerWorker[task.parTblIdx] = hMap
 		}
 		var mapDelta int64
 		var rowDelta int64
@@ -844,11 +846,57 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition() {
 			w.memUsageDelta += mapDelta + (rowDelta * int64(unsafe.Sizeof(reflect.Int64)))
 			w.totalMemUsage += w.memUsageDelta
 		}
-		if w.rowDelta >= int64(w.memConsumeBatchSize) {
+		if w.rowDelta >= int64(w.batchSize) {
 			w.consumeMemDelta()
 		}
 		failpoint.Inject("testIndexMergeIntersectionWorkerPanic", nil)
 	}
+
+	// We assume the result of intersection is small, so no need to track memory.
+	intersecteds := make(map[int][]kv.Handle, len(w.handleMapsPerWorker))
+	for parTblIdx, hMap := range w.handleMapsPerWorker {
+		hMap.Range(func(h kv.Handle, val interface{}) bool {
+			if *(val.(*int)) == len(w.indexMerge.partialPlans) {
+				// Means all partial paths have this handle.
+				intersecteds[parTblIdx] = append(intersecteds[parTblIdx], h)
+			}
+			return true
+		})
+	}
+
+	tasks := make([]*indexMergeTableTask, 0, len(w.handleMapsPerWorker))
+	for parTblIdx, intersected := range intersecteds {
+		// Split intersected[parTblIdx] to avoid task is too large.
+		for len(intersected) > 0 {
+			length := w.batchSize
+			if length > len(intersected) {
+				length = len(intersected)
+			}
+			task := &indexMergeTableTask{
+				lookupTableTask: lookupTableTask{
+					handles: intersected[:length],
+					doneCh:  make(chan error, 1),
+				},
+			}
+			intersected = intersected[length:]
+			if w.indexMerge.partitionTableMode {
+				task.partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
+			}
+			tasks = append(tasks, task)
+			logutil.BgLogger().Debug("intersectionProcessWorker build tasks",
+				zap.Int("parTblIdx", parTblIdx), zap.Int("task.handles", len(task.handles)))
+		}
+	}
+	for _, task := range tasks {
+		select {
+		case <-ctx.Done():
+			return
+		case <-finished:
+			return
+		case workCh <- task:
+			resultCh <- task
+		}
+	}
 }
 
 // For each partition(dynamic mode), a map is used to do intersection. Key of the map is handle, and value is the number of times it occurs.
@@ -870,22 +918,17 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 
 	// One goroutine may handle one or multiple partitions.
 	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
-	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
 	partCnt := 1
 	workerCnt := 1
-	partCntPerWorker := 1
-
+	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
 	const maxChannelSize int = 1024
+
 	batchSize := w.indexMerge.ctx.GetSessionVars().IndexLookupSize
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
 		workerCnt = partCnt
 		if workerCnt > maxWorkerCnt {
 			workerCnt = maxWorkerCnt
-			partCntPerWorker = partCnt / workerCnt
-			if partCnt%workerCnt != 0 {
-				partCntPerWorker++
-			}
 		}
 	}
 	failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
@@ -894,25 +937,18 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			panic(fmt.Sprintf("unexpected workerCnt, expect %d, got %d", con, workerCnt))
 		}
 	})
-	handleMaps := make([]*kv.MemAwareHandleMap[*int], 0, partCnt)
+
 	workers := make([]*intersectionProcessWorker, 0, workerCnt)
 	wg := sync.WaitGroup{}
 	errCh := make(chan bool, workerCnt)
 	for i := 0; i < workerCnt; i++ {
-		handleMapsPerWorker := make(map[int]*kv.MemAwareHandleMap[*int], partCntPerWorker)
-		for j := 0; j < partCntPerWorker; j++ {
-			hMap := kv.NewMemAwareHandleMap[*int]()
-			handleMaps = append(handleMaps, hMap)
-			parTblIdx := i*partCntPerWorker + j
-			handleMapsPerWorker[parTblIdx] = hMap
-		}
 		worker := &intersectionProcessWorker{
 			workerID:            i,
-			handleMapsPerWorker: handleMapsPerWorker,
+			handleMapsPerWorker: make(map[int]*kv.MemAwareHandleMap[*int]),
 			workerCh:            make(chan *indexMergeTableTask, maxChannelSize),
 			indexMerge:          w.indexMerge,
 			memTracker:          w.indexMerge.memTracker,
-			memConsumeBatchSize: batchSize,
+			batchSize:           batchSize,
 		}
 		wg.Add(1)
 		go func() {
@@ -920,7 +956,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			defer wg.Done()
 			util.WithRecovery(
 				func() {
-					worker.doIntersectionPerPartition()
+					worker.doIntersectionPerPartition(ctx, workCh, resultCh, finished)
 				},
 				w.handleLoopFetcherPanic(ctx, resultCh, "IndexMergeIntersectionProcessWorker", errCh),
 			)
@@ -930,7 +966,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	var processWorkerErr bool
 	for task := range fetchCh {
 		select {
-		case workers[task.parTblIdx/partCntPerWorker].workerCh <- task:
+		case workers[task.parTblIdx%workerCnt].workerCh <- task:
 		case processWorkerErr = <-errCh:
 			break
 		}
@@ -951,54 +987,6 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		w.indexMerge.memTracker.Consume(-(totalMemUsage))
 	}()
 
-	// We assume the result of intersection is small, so no need to track memory.
-	intersected := make([][]kv.Handle, partCnt)
-	for parTblIdx, hMap := range handleMaps {
-		hMap.Range(func(h kv.Handle, val interface{}) bool {
-			if *(val.(*int)) == len(w.indexMerge.partialPlans) {
-				// Means all partial paths have this handle.
-				intersected[parTblIdx] = append(intersected[parTblIdx], h)
-			}
-			return true
-		})
-	}
-	tasks := make([]*indexMergeTableTask, 0, partCnt)
-	for parTblIdx := 0; parTblIdx < partCnt; parTblIdx++ {
-		if len(intersected[parTblIdx]) == 0 {
-			continue
-		}
-		// Split intersected[parTblIdx] to avoid task is too large.
-		for len(intersected[parTblIdx]) > 0 {
-			length := batchSize
-			if length > len(intersected[parTblIdx]) {
-				length = len(intersected[parTblIdx])
-			}
-			s := intersected[parTblIdx][:length]
-			intersected[parTblIdx] = intersected[parTblIdx][length:]
-			task := &indexMergeTableTask{
-				lookupTableTask: lookupTableTask{
-					handles: s,
-					doneCh:  make(chan error, 1),
-				},
-			}
-			if w.indexMerge.partitionTableMode {
-				task.partitionTable = w.indexMerge.prunedPartitions[parTblIdx]
-			}
-			tasks = append(tasks, task)
-			logutil.BgLogger().Debug("intersectionProcessWorker build tasks",
-				zap.Int("parTblIdx", parTblIdx), zap.Int("task.handles", len(task.handles)))
-		}
-	}
-	for _, task := range tasks {
-		select {
-		case <-ctx.Done():
-			return
-		case <-finished:
-			return
-		case workCh <- task:
-			resultCh <- task
-		}
-	}
 }
 
 func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask, worker string, extraCh chan bool) func(r interface{}) {

From 29c3ebd68327353774772ae40b85ee9e27109868 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 15:24:30 +0800
Subject: [PATCH 27/33] use WaitGroupWrapper

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 21645f3060b61..6b30f7b3f16d4 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -921,7 +921,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	partCnt := 1
 	workerCnt := 1
 	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
-	const maxChannelSize int = 1024
+	maxChannelSize := atomic.LoadInt32(&LookupTableTaskChannelSize)
 
 	batchSize := w.indexMerge.ctx.GetSessionVars().IndexLookupSize
 	if w.indexMerge.partitionTableMode {
@@ -939,7 +939,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	})
 
 	workers := make([]*intersectionProcessWorker, 0, workerCnt)
-	wg := sync.WaitGroup{}
+	wg := util.WaitGroupWrapper{}
 	errCh := make(chan bool, workerCnt)
 	for i := 0; i < workerCnt; i++ {
 		worker := &intersectionProcessWorker{
@@ -950,17 +950,10 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 			memTracker:          w.indexMerge.memTracker,
 			batchSize:           batchSize,
 		}
-		wg.Add(1)
-		go func() {
+		wg.RunWithRecover(func() {
 			defer trace.StartRegion(ctx, "IndexMergeIntersectionProcessWorker").End()
-			defer wg.Done()
-			util.WithRecovery(
-				func() {
-					worker.doIntersectionPerPartition(ctx, workCh, resultCh, finished)
-				},
-				w.handleLoopFetcherPanic(ctx, resultCh, "IndexMergeIntersectionProcessWorker", errCh),
-			)
-		}()
+			worker.doIntersectionPerPartition(ctx, workCh, resultCh, finished)
+		}, w.handleLoopFetcherPanic(ctx, resultCh, "IndexMergeIntersectionProcessWorker", errCh))
 		workers = append(workers, worker)
 	}
 	var processWorkerErr bool

From 3bf600e722daf6ccb87c6f0a0f1fd3d19a26424a Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 15:29:02 +0800
Subject: [PATCH 28/33] remove defer for last consume

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 6b30f7b3f16d4..d88bd2c63ba24 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -976,10 +976,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	for _, processWorker := range workers {
 		totalMemUsage += processWorker.totalMemUsage
 	}
-	defer func() {
-		w.indexMerge.memTracker.Consume(-(totalMemUsage))
-	}()
-
+	w.indexMerge.memTracker.Consume(-(totalMemUsage))
 }
 
 func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask, worker string, extraCh chan bool) func(r interface{}) {

From 7486eff4f9b6907af360bf353a163ec2f2f8be9c Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 15:45:44 +0800
Subject: [PATCH 29/33] change var name

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index d88bd2c63ba24..3df2235993b8f 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -830,6 +830,7 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Conte
 		var mapDelta int64
 		var rowDelta int64
 		for _, h := range task.handles {
+			// Use *int to avoid Get() again.
 			if cntPtr, ok := hMap.Get(h); ok {
 				(*cntPtr)++
 			} else {
@@ -853,19 +854,19 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Conte
 	}
 
 	// We assume the result of intersection is small, so no need to track memory.
-	intersecteds := make(map[int][]kv.Handle, len(w.handleMapsPerWorker))
+	intersectedMap := make(map[int][]kv.Handle, len(w.handleMapsPerWorker))
 	for parTblIdx, hMap := range w.handleMapsPerWorker {
 		hMap.Range(func(h kv.Handle, val interface{}) bool {
 			if *(val.(*int)) == len(w.indexMerge.partialPlans) {
 				// Means all partial paths have this handle.
-				intersecteds[parTblIdx] = append(intersecteds[parTblIdx], h)
+				intersectedMap[parTblIdx] = append(intersectedMap[parTblIdx], h)
 			}
 			return true
 		})
 	}
 
 	tasks := make([]*indexMergeTableTask, 0, len(w.handleMapsPerWorker))
-	for parTblIdx, intersected := range intersecteds {
+	for parTblIdx, intersected := range intersectedMap {
 		// Split intersected[parTblIdx] to avoid task is too large.
 		for len(intersected) > 0 {
 			length := w.batchSize

From f7a4bcb22e28de0004026ffb4ecc4365f584a531 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 15:50:11 +0800
Subject: [PATCH 30/33] fix lint

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index 3df2235993b8f..c98f6535f3fb9 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -919,8 +919,8 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 
 	// One goroutine may handle one or multiple partitions.
 	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
-	partCnt := 1
-	workerCnt := 1
+	var partCnt int
+	var workerCnt int
 	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
 	maxChannelSize := atomic.LoadInt32(&LookupTableTaskChannelSize)
 
@@ -931,6 +931,9 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		if workerCnt > maxWorkerCnt {
 			workerCnt = maxWorkerCnt
 		}
+	} else {
+		partCnt = 1
+		workerCnt = 1
 	}
 	failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
 		con := val.(int)

From 8cca18a73c46731cdb47cda05dcdc9772ed47965 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 16:42:03 +0800
Subject: [PATCH 31/33] fix lint

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index c98f6535f3fb9..f1e223bba2c7b 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -919,21 +919,17 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 
 	// One goroutine may handle one or multiple partitions.
 	// Max number of partition number is 8192, we use ExecutorConcurrency to avoid too many goroutines.
-	var partCnt int
-	var workerCnt int
 	maxWorkerCnt := w.indexMerge.ctx.GetSessionVars().IndexMergeIntersectionConcurrency()
 	maxChannelSize := atomic.LoadInt32(&LookupTableTaskChannelSize)
-
 	batchSize := w.indexMerge.ctx.GetSessionVars().IndexLookupSize
+
+	partCnt := 1
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
-		workerCnt = partCnt
-		if workerCnt > maxWorkerCnt {
-			workerCnt = maxWorkerCnt
-		}
-	} else {
-		partCnt = 1
-		workerCnt = 1
+	}
+	workerCnt := partCnt
+	if workerCnt > maxWorkerCnt {
+		workerCnt = maxWorkerCnt
 	}
 	failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
 		con := val.(int)

From 5cfbe24aa4531d001257e6651680bcd4004dd5cb Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 17:16:26 +0800
Subject: [PATCH 32/33] fix comment; update usage of memtracker

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 45 ++++++++++++----------------------
 1 file changed, 15 insertions(+), 30 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index f1e223bba2c7b..b487089cfa30c 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"reflect"
 	"runtime/trace"
 	"sync"
 	"sync/atomic"
@@ -799,26 +798,19 @@ type intersectionProcessWorker struct {
 	memTracker          *memory.Tracker
 	batchSize           int
 
-	// When rowDelta == memConsumeBatchSize, Consume(memUsageDelta)
-	memUsageDelta int64
+	// When rowDelta == memConsumeBatchSize, Consume(memUsage)
 	rowDelta      int64
-
-	// For Consume(-totalMemUsage)
-	totalMemUsage int64
+	mapUsageDelta int64
 }
 
 func (w *intersectionProcessWorker) consumeMemDelta() {
-	w.memTracker.Consume(w.memUsageDelta)
-	w.memUsageDelta = 0
+	w.memTracker.Consume(w.mapUsageDelta + w.rowDelta*int64(unsafe.Sizeof(int(0))))
+	w.mapUsageDelta = 0
 	w.rowDelta = 0
 }
 
 func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Context, workCh chan<- *indexMergeTableTask, resultCh chan<- *indexMergeTableTask, finished <-chan struct{}) {
-	defer func() {
-		if w.rowDelta > 0 {
-			w.consumeMemDelta()
-		}
-	}()
+	defer w.memTracker.Detach()
 
 	for task := range w.workerCh {
 		var ok bool
@@ -842,16 +834,17 @@ func (w *intersectionProcessWorker) doIntersectionPerPartition(ctx context.Conte
 
 		logutil.BgLogger().Debug("intersectionProcessWorker handle tasks", zap.Int("workerID", w.workerID),
 			zap.Int("task.handles", len(task.handles)), zap.Int64("rowDelta", rowDelta))
-		if rowDelta > 0 {
-			w.rowDelta += rowDelta
-			w.memUsageDelta += mapDelta + (rowDelta * int64(unsafe.Sizeof(reflect.Int64)))
-			w.totalMemUsage += w.memUsageDelta
-		}
+
+		w.mapUsageDelta += mapDelta
+		w.rowDelta += rowDelta
 		if w.rowDelta >= int64(w.batchSize) {
 			w.consumeMemDelta()
 		}
 		failpoint.Inject("testIndexMergeIntersectionWorkerPanic", nil)
 	}
+	if w.rowDelta > 0 {
+		w.consumeMemDelta()
+	}
 
 	// We assume the result of intersection is small, so no need to track memory.
 	intersectedMap := make(map[int][]kv.Handle, len(w.handleMapsPerWorker))
@@ -942,12 +935,14 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	wg := util.WaitGroupWrapper{}
 	errCh := make(chan bool, workerCnt)
 	for i := 0; i < workerCnt; i++ {
+		tracker := memory.NewTracker(w.indexMerge.id, -1)
+		tracker.AttachTo(w.indexMerge.memTracker)
 		worker := &intersectionProcessWorker{
 			workerID:            i,
 			handleMapsPerWorker: make(map[int]*kv.MemAwareHandleMap[*int]),
 			workerCh:            make(chan *indexMergeTableTask, maxChannelSize),
 			indexMerge:          w.indexMerge,
-			memTracker:          w.indexMerge.memTracker,
+			memTracker:          tracker,
 			batchSize:           batchSize,
 		}
 		wg.RunWithRecover(func() {
@@ -956,11 +951,10 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		}, w.handleLoopFetcherPanic(ctx, resultCh, "IndexMergeIntersectionProcessWorker", errCh))
 		workers = append(workers, worker)
 	}
-	var processWorkerErr bool
 	for task := range fetchCh {
 		select {
 		case workers[task.parTblIdx%workerCnt].workerCh <- task:
-		case processWorkerErr = <-errCh:
+		case <-errCh:
 			break
 		}
 	}
@@ -968,15 +962,6 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 		close(processWorker.workerCh)
 	}
 	wg.Wait()
-	if processWorkerErr {
-		return
-	}
-
-	var totalMemUsage int64
-	for _, processWorker := range workers {
-		totalMemUsage += processWorker.totalMemUsage
-	}
-	w.indexMerge.memTracker.Consume(-(totalMemUsage))
 }
 
 func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, resultCh chan<- *indexMergeTableTask, worker string, extraCh chan bool) func(r interface{}) {

From 3e5414e39337f8aadcf51255832ed6f8e88028e9 Mon Sep 17 00:00:00 2001
From: guo-shaoge <shaoge1994@163.com>
Date: Tue, 29 Nov 2022 21:46:28 +0800
Subject: [PATCH 33/33] use mathutil

Signed-off-by: guo-shaoge <shaoge1994@163.com>
---
 executor/index_merge_reader.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go
index b487089cfa30c..71220b53a8258 100644
--- a/executor/index_merge_reader.go
+++ b/executor/index_merge_reader.go
@@ -920,10 +920,7 @@ func (w *indexMergeProcessWorker) fetchLoopIntersection(ctx context.Context, fet
 	if w.indexMerge.partitionTableMode {
 		partCnt = len(w.indexMerge.prunedPartitions)
 	}
-	workerCnt := partCnt
-	if workerCnt > maxWorkerCnt {
-		workerCnt = maxWorkerCnt
-	}
+	workerCnt := mathutil.Min(partCnt, maxWorkerCnt)
 	failpoint.Inject("testIndexMergeIntersectionConcurrency", func(val failpoint.Value) {
 		con := val.(int)
 		if con != workerCnt {