store/tikv: fix a concurrency bug that may cause the batchClient time…

…out (#22239) (#22336)

Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

store/tikv/client.go

@@ -221,6 +221,9 @@ type rpcClient struct {
 	security config.Security
 
 	idleNotify uint32
+	// recycleMu protect the conns from being modified during a connArray is taken out and used.
+	// That means recycleIdleConnArray() will wait until nobody doing sendBatchRequest()
+	recycleMu sync.RWMutex
 	// Periodically check whether there is any connection that is idle and then close and remove these connections.
 	// Implement background cleanup.
 	isClosed    bool
@@ -336,11 +339,15 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R
 	}()
 
 	if atomic.CompareAndSwapUint32(&c.idleNotify, 1, 0) {
+		c.recycleMu.Lock()
 		c.recycleIdleConnArray()
+		c.recycleMu.Unlock()
 	}
 
 	// TiDB will not send batch commands to TiFlash, to resolve the conflict with Batch Cop Request.
 	enableBatch := req.StoreTp != kv.TiDB && req.StoreTp != kv.TiFlash
+	c.recycleMu.RLock()
+	defer c.recycleMu.RUnlock()
 	connArray, err := c.getConnArray(addr, enableBatch)
 	if err != nil {
 		return nil, errors.Trace(err)