Orphan pods cleaner

pkg/controller/tidbcluster/tidb_cluster_control.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/manager"
+	"github.com/pingcap/tidb-operator/pkg/manager/member"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/tools/record"
@@ -39,6 +40,7 @@ func NewDefaultTidbClusterControl(
 	tidbMemberManager manager.Manager,
 	reclaimPolicyManager manager.Manager,
 	metaManager manager.Manager,
+	orphanPodsCleaner member.OrphanPodsCleaner,
 	recorder record.EventRecorder) ControlInterface {
 	return &defaultTidbClusterControl{
 		tcControl,
@@ -47,6 +49,7 @@ func NewDefaultTidbClusterControl(
 		tidbMemberManager,
 		reclaimPolicyManager,
 		metaManager,
+		orphanPodsCleaner,
 		recorder,
 	}
 }
@@ -58,6 +61,7 @@ type defaultTidbClusterControl struct {
 	tidbMemberManager    manager.Manager
 	reclaimPolicyManager manager.Manager
 	metaManager          manager.Manager
+	orphanPodsCleaner    member.OrphanPodsCleaner
 	recorder             record.EventRecorder
 }
 
@@ -92,6 +96,11 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 		return err
 	}
 
+	_, err = tcc.orphanPodsCleaner.Clean(tc)
+	if err != nil {
+		return err
+	}
+
 	// PD
 	err = tcc.pdMemberManager.Sync(tc)
 	if err != nil {

pkg/controller/tidbcluster/tidb_cluster_control_test.go

@@ -361,7 +361,8 @@ func newFakeTidbClusterControl() (ControlInterface, *meta.FakeReclaimPolicyManag
 	tidbMemberManager := mm.NewFakeTiDBMemberManager()
 	reclaimPolicyManager := meta.NewFakeReclaimPolicyManager()
 	metaManager := meta.NewFakeMetaManager()
-	control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, recorder)
+	opc := mm.NewFakeOrphanPodsCleaner()
+	control := NewDefaultTidbClusterControl(tcControl, pdMemberManager, tikvMemberManager, tidbMemberManager, reclaimPolicyManager, metaManager, opc, recorder)
 
 	return control, reclaimPolicyManager, pdMemberManager, tikvMemberManager, tidbMemberManager, metaManager
 }

pkg/controller/tidbcluster/tidb_cluster_controller.go

@@ -162,6 +162,11 @@ func NewController(
 				podInformer.Lister(),
 				podControl,
 			),
+			mm.NewOrphanPodsCleaner(
+				podInformer.Lister(),
+				podControl,
+				pvcInformer.Lister(),
+			),
 			recorder,
 		),
 		queue: workqueue.NewNamedRateLimitingQueue(

pkg/controller/tidbcluster/tidb_cluster_controller_test.go

@@ -319,6 +319,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, cache.Indexer)
 			podInformer.Lister(),
 			podControl,
 		),
+		mm.NewFakeOrphanPodsCleaner(),
 		recorder,
 	)
 

pkg/label/label_test.go

@@ -16,9 +16,8 @@ package label
 import (
 	"testing"
 
-	"k8s.io/apimachinery/pkg/labels"
-
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/labels"
 )
 
 func TestLabelNew(t *testing.T) {

pkg/manager/member/orphan_pods_cleaner.go

@@ -0,0 +1,109 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package member
+
+import (
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
+	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/label"
+	"k8s.io/apimachinery/pkg/api/errors"
+	corelisters "k8s.io/client-go/listers/core/v1"
+)
+
+const (
+	skipReasonOrphanPodsCleanerIsNotPDOrTiKV  = "orphan pods cleaner: member type is not pd or tikv"
+	skipReasonOrphanPodsCleanerPVCNameIsEmpty = "orphan pods cleaner: pvcName is empty"
+	skipReasonOrphanPodsCleanerPVCIsFound     = "orphan pods cleaner: pvc is found"
+)
+
+// OrphanPodsCleaner implements the logic for cleaning the orphan pods(has no pvc)
+type OrphanPodsCleaner interface {
+	Clean(*v1alpha1.TidbCluster) (map[string]string, error)
+}
+
+type orphanPodsCleaner struct {
+	podLister  corelisters.PodLister
+	podControl controller.PodControlInterface
+	pvcLister  corelisters.PersistentVolumeClaimLister
+}
+
+// NewOrphanPodsCleaner returns a OrphanPodsCleaner
+func NewOrphanPodsCleaner(podLister corelisters.PodLister,
+	podControl controller.PodControlInterface,
+	pvcLister corelisters.PersistentVolumeClaimLister) OrphanPodsCleaner {
+	return &orphanPodsCleaner{podLister, podControl, pvcLister}
+}
+
+func (opc *orphanPodsCleaner) Clean(tc *v1alpha1.TidbCluster) (map[string]string, error) {
+	ns := tc.GetNamespace()
+	// for unit test
+	skipReason := map[string]string{}
+
+	selector, err := label.New().Instance(tc.GetLabels()[label.InstanceLabelKey]).Selector()
+	if err != nil {
+		return skipReason, err
+	}
+	pods, err := opc.podLister.Pods(ns).List(selector)
+	if err != nil {
+		return skipReason, err
+	}
+
+	for _, pod := range pods {
+		podName := pod.GetName()
+		l := label.Label(pod.Labels)
+		if !(l.IsPD() || l.IsTiKV()) {
+			skipReason[podName] = skipReasonOrphanPodsCleanerIsNotPDOrTiKV
+			continue
+		}
+
+		var pvcName string
+		for _, vol := range pod.Spec.Volumes {
+			if vol.PersistentVolumeClaim != nil {
+				pvcName = vol.PersistentVolumeClaim.ClaimName
+				break
+			}
+		}
+		if pvcName == "" {
+			skipReason[podName] = skipReasonOrphanPodsCleanerPVCNameIsEmpty
+			continue
+		}
+
+		_, err := opc.pvcLister.PersistentVolumeClaims(ns).Get(pvcName)
+		if err == nil {
+			skipReason[podName] = skipReasonOrphanPodsCleanerPVCIsFound
+			continue
+		}
+		if !errors.IsNotFound(err) {
+			return skipReason, err
+		}
+
+		err = opc.podControl.DeletePod(tc, pod)
+		if err != nil {
+			return skipReason, err
+		}
+	}
+
+	return skipReason, nil
+}
+
+type fakeOrphanPodsCleaner struct{}
+
+// NewFakeOrphanPodsCleaner returns a fake orphan pods cleaner
+func NewFakeOrphanPodsCleaner() OrphanPodsCleaner {
+	return &fakeOrphanPodsCleaner{}
+}
+
+func (fopc *fakeOrphanPodsCleaner) Clean(_ *v1alpha1.TidbCluster) (map[string]string, error) {
+	return nil, nil
+}