diff --git a/executor/index_lookup_hash_join.go b/executor/index_lookup_hash_join.go
index 11f1298708797..65ec964ed2d55 100644
--- a/executor/index_lookup_hash_join.go
+++ b/executor/index_lookup_hash_join.go
@@ -483,6 +483,10 @@ func (iw *indexHashJoinInnerWorker) run(ctx context.Context, cancelFunc context.
 	}
 	h, resultCh := fnv.New64(), iw.resultCh
 	for {
+		// The previous task has been processed, so release the occupied memory
+		if task != nil {
+			task.memTracker.Detach()
+		}
 		select {
 		case <-ctx.Done():
 			return
diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go
index fedf167813520..b06de10a06ce5 100644
--- a/executor/index_lookup_join.go
+++ b/executor/index_lookup_join.go
@@ -332,6 +332,10 @@ func (e *IndexLookUpJoin) getFinishedTask(ctx context.Context) (*lookUpJoinTask,
 		return task, nil
 	}
 
+	// The previous task has been processed, so release the occupied memory
+	if task != nil {
+		task.memTracker.Detach()
+	}
 	select {
 	case task = <-e.resultCh:
 	case <-ctx.Done():
@@ -573,7 +577,7 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi
 				return nil, err
 			}
 			if rowIdx == 0 {
-				iw.lookup.memTracker.Consume(types.EstimatedMemUsage(dLookUpKey, numRows))
+				iw.memTracker.Consume(types.EstimatedMemUsage(dLookUpKey, numRows))
 			}
 			if dHashKey == nil {
 				// Append null to make looUpKeys the same length as outer Result.