Clean pod schedule annotation

pkg/controller/tidbcluster/tidb_cluster_control.go

@@ -41,6 +41,7 @@ func NewDefaultTidbClusterControl(
 	reclaimPolicyManager manager.Manager,
 	metaManager manager.Manager,
 	orphanPodsCleaner member.OrphanPodsCleaner,
+	pvcCleaner member.PVCCleanerInterface,
 	recorder record.EventRecorder) ControlInterface {
 	return &defaultTidbClusterControl{
 		tcControl,
@@ -50,6 +51,7 @@ func NewDefaultTidbClusterControl(
 		reclaimPolicyManager,
 		metaManager,
 		orphanPodsCleaner,
+		pvcCleaner,
 		recorder,
 	}
 }
@@ -62,6 +64,7 @@ type defaultTidbClusterControl struct {
 	reclaimPolicyManager manager.Manager
 	metaManager          manager.Manager
 	orphanPodsCleaner    member.OrphanPodsCleaner
+	pvcCleaner           member.PVCCleanerInterface
 	recorder             record.EventRecorder
 }
 
@@ -138,5 +141,11 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 	//   - label.StoreIDLabelKey
 	//   - label.MemberIDLabelKey
 	//   - label.NamespaceLabelKey
-	return tcc.metaManager.Sync(tc)
+	if err := tcc.metaManager.Sync(tc); err != nil {
+		return err
+	}
+
+	// cleaning the pod scheduling annotation for pd and tikv
+	_, err := tcc.pvcCleaner.Clean(tc)
+	return err
 }

pkg/controller/tidbcluster/tidb_cluster_control_test.go

@@ -232,7 +232,8 @@ func newFakeTidbClusterControl() (ControlInterface, *meta.FakeReclaimPolicyManag
 	reclaimPolicyManager := meta.NewFakeReclaimPolicyManager()
 	metaManager := meta.NewFakeMetaManager()
 	opc := mm.NewFakeOrphanPodsCleaner()
-	control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, opc, recorder)
+	pcc := mm.NewFakePVCCleaner()
+	control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, opc, pcc, recorder)
 
 	return control, reclaimPolicyManager, pdMemberManager, tikvMemberManager, tidbMemberManager, metaManager
 }

pkg/controller/tidbcluster/tidb_cluster_controller.go

@@ -172,6 +172,11 @@ func NewController(
 				pvcInformer.Lister(),
 				kubeCli,
 			),
+			mm.NewRealPVCCleaner(
+				podInformer.Lister(),
+				pvcControl,
+				pvcInformer.Lister(),
+			),
 			recorder,
 		),
 		queue: workqueue.NewNamedRateLimitingQueue(

pkg/controller/tidbcluster/tidb_cluster_controller_test.go

@@ -324,6 +324,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer)
 			podControl,
 		),
 		mm.NewFakeOrphanPodsCleaner(),
+		mm.NewFakePVCCleaner(),
 		recorder,
 	)
 

pkg/manager/member/pvc_cleaner.go

@@ -0,0 +1,152 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package member
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
+	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/label"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	corelisters "k8s.io/client-go/listers/core/v1"
+)
+
+const (
+	skipReasonPVCCleanerIsNotPDOrTiKV            = "pvc cleaner: member type is not pd or tikv"
+	skipReasonPVCCleanerDeferDeletePVCNotHasLock = "pvc cleaner: defer delete PVC not has schedule lock"
+	skipReasonPVCCleanerPVCNotHasLock            = "pvc cleaner: pvc not has schedule lock"
+	skipReasonPVCCleanerPodWaitingForScheduling  = "pvc cleaner: waiting for pod scheduling"
+	skipReasonPVCCleanerPodNotFound              = "pvc cleaner: the corresponding pod of pvc has not been found"
+	skipReasonPVCCleanerWaitingForPVCSync        = "pvc cleaner: waiting for pvc's meta info to be synced"
+)
+
+// PVCCleaner implements the logic for cleaning the pvc related resource
+type PVCCleanerInterface interface {
+	Clean(*v1alpha1.TidbCluster) (map[string]string, error)
+}
+
+type realPVCCleaner struct {
+	podLister  corelisters.PodLister
+	pvcControl controller.PVCControlInterface
+	pvcLister  corelisters.PersistentVolumeClaimLister
+}
+
+// NewRealPVCCleaner returns a realPVCCleaner
+func NewRealPVCCleaner(
+	podLister corelisters.PodLister,
+	pvcControl controller.PVCControlInterface,
+	pvcLister corelisters.PersistentVolumeClaimLister) PVCCleanerInterface {
+	return &realPVCCleaner{
+		podLister,
+		pvcControl,
+		pvcLister,
+	}
+}
+
+func (rpc *realPVCCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, error) {
+	ns := tc.GetNamespace()
+	tcName := tc.GetName()
+	// for unit test
+	skipReason := map[string]string{}
+
+	selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector()
+	if err != nil {
+		return skipReason, fmt.Errorf("cluster %s/%s assemble label selector failed, err: %v", ns, tcName, err)
+	}
+
+	pvcs, err := rpc.pvcLister.PersistentVolumeClaims(ns).List(selector)
+	if err != nil {
+		return skipReason, fmt.Errorf("cluster %s/%s list pvc failed, selector: %s, err: %v", ns, tcName, selector, err)
+	}
+
+	for _, pvc := range pvcs {
+		pvcName := pvc.GetName()
+		l := label.Label(pvc.Labels)
+		if !(l.IsPD() || l.IsTiKV()) {
+			skipReason[pvcName] = skipReasonPVCCleanerIsNotPDOrTiKV
+			continue
+		}
+
+		if pvc.Annotations[label.AnnPVCDeferDeleting] != "" {
+			if _, exist := pvc.Annotations[label.AnnPVCPodScheduling]; !exist {
+				// The defer deleting PVC without pod scheduling annotation, do nothing
+				glog.V(4).Infof("cluster %s/%s defer delete pvc %s has not pod scheduling annotation, skip clean", ns, tcName, pvcName)
+				skipReason[pvcName] = skipReasonPVCCleanerDeferDeletePVCNotHasLock
+				continue
+			}
+			// The defer deleting PVC has pod scheduling annotation, so we need to delete the pod scheduling annotation
+			delete(pvc.Annotations, label.AnnPVCPodScheduling)
+			if _, err := rpc.pvcControl.UpdatePVC(tc, pvc); err != nil {
+				return skipReason, fmt.Errorf("cluster %s/%s remove pvc %s pod scheduling annotation faild, err: %v", ns, tcName, pvcName, err)
+			}
+			continue
+		}
+
+		podName, exist := pvc.Annotations[label.AnnPodNameKey]
+		if !exist {
+			// waiting for pvc's meta info to be synced
+			skipReason[pvcName] = skipReasonPVCCleanerWaitingForPVCSync
+			continue
+		}
+
+		pod, err := rpc.podLister.Pods(ns).Get(podName)
+		if err != nil {
+			if !errors.IsNotFound(err) {
+				return skipReason, fmt.Errorf("cluster %s/%s get pvc %s pod %s failed, err: %v", ns, tcName, pvcName, podName, err)
+			}
+			skipReason[pvcName] = skipReasonPVCCleanerPodNotFound
+			continue
+		}
+
+		if _, exist := pvc.Annotations[label.AnnPVCPodScheduling]; !exist {
+			// The PVC without pod scheduling annotation, do nothing
+			glog.V(4).Infof("cluster %s/%s pvc %s has not pod scheduling annotation, skip clean", ns, tcName, pvcName)
+			skipReason[pvcName] = skipReasonPVCCleanerPVCNotHasLock
+			continue
+		}
+
+		if pvc.Status.Phase != corev1.ClaimBound || pod.Spec.NodeName == "" {
+			// This pod has not been scheduled yet, no need to clean up the pvc pod schedule annotation
+			glog.V(4).Infof("cluster %s/%s pod %s has not been scheduled yet, skip clean pvc %s pod schedule annotation", ns, tcName, podName, pvcName)
+			skipReason[pvcName] = skipReasonPVCCleanerPodWaitingForScheduling
+			continue
+		}
+
+		delete(pvc.Annotations, label.AnnPVCPodScheduling)
+		if _, err := rpc.pvcControl.UpdatePVC(tc, pvc); err != nil {
+			return skipReason, fmt.Errorf("cluster %s/%s remove pvc %s pod scheduling annotation faild, err: %v", ns, tcName, pvcName, err)
+		}
+		glog.Infof("cluster %s/%s, clean pvc %s pod scheduling annotation successfully", ns, tcName, pvcName)
+	}
+
+	return skipReason, nil
+}
+
+var _ PVCCleanerInterface = &realPVCCleaner{}
+
+type fakePVCCleaner struct{}
+
+// NewFakePVCCleaner returns a fake PVC cleaner
+func NewFakePVCCleaner() PVCCleanerInterface {
+	return &fakePVCCleaner{}
+}
+
+func (fpc *fakePVCCleaner) Clean(_ *v1alpha1.TidbCluster) (map[string]string, error) {
+	return nil, nil
+}
+
+var _ PVCCleanerInterface = &fakePVCCleaner{}