Delay removal of orphanPVC to avoid the removal of PVC in use

controllers/druid/handler.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"regexp"
 	"sort"
+	"strconv"
+	"time"
 
 	autoscalev2 "k8s.io/api/autoscaling/v2"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -490,12 +492,65 @@ func deleteOrphanPVC(ctx context.Context, sdk client.Client, drd *v1alpha1.Druid
 		for i, pvc := range pvcList {
 
 			if !ContainsString(mountedPVC, pvc.GetName()) {
-				err := writers.Delete(ctx, sdk, drd, pvcList[i], emitEvents, &client.DeleteAllOfOptions{})
-				if err != nil {
-					return err
+
+				if _, ok := pvc.GetLabels()["toBeDeleted"]; ok {
+					deletionTS := pvc.GetLabels()["deletionTimestamp"]
+
+					deletionTimestamp, err := strconv.ParseInt(deletionTS, 10, 64)
+
+					if err != nil {
+						msg := fmt.Sprintf("Unable to parse label deletionTimestamp [%s:%s]", deletionTS, pvcList[i].GetName())
+						logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+						return err
+					}
+
+					timeNow := time.Now().Unix()
+					timeDiff := timeDifference(deletionTimestamp, timeNow)
+
+					if timeDiff >= int64(time.Second/time.Second)*60 {
+						// delete pvc
+						err = writers.Delete(ctx, sdk, drd, pvcList[i], emitEvents, &client.DeleteAllOfOptions{})
+						if err != nil {
+							return err
+						} else {
+							msg := fmt.Sprintf("Deleted orphaned pvc [%s:%s] successfully", pvcList[i].GetName(), drd.Namespace)
+							logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+						}
+					} else {
+						// wait for 60s
+						msg := fmt.Sprintf("pvc [%s:%s] marked to be deleted after %ds", pvcList[i].GetName(), drd.Namespace, timeDiff)
+						logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+					}
 				} else {
-					msg := fmt.Sprintf("Deleted orphaned pvc [%s:%s] successfully", pvcList[i].GetName(), drd.Namespace)
-					logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+					// set labels when pvc comes for deletion for the first time
+					getPvcLabels := pvc.GetLabels()
+					getPvcLabels["toBeDeleted"] = "yes"
+					getPvcLabels["deletionTimestamp"] = strconv.FormatInt(time.Now().Unix(), 10)
+					pvc.SetLabels(getPvcLabels)
+
+					_, err = writers.Update(ctx, sdk, drd, pvc, emitEvents)
+					if err != nil {
+						return err
+					} else {
+						msg := fmt.Sprintf("marked pvc for deletion , added labels %s and %s successfully [%s]", "toBeDeleted", "deletionTimestamp", pvc.GetName())
+						logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+					}
+				}
+			} else {
+				// do not delete pvc
+				if _, ok := pvc.GetLabels()["toBeDeleted"]; ok {
+					getPvcLabels := pvc.GetLabels()
+					delete(getPvcLabels, "toBeDeleted")
+					delete(getPvcLabels, "deletionTimestamp")
+					pvc.SetLabels(getPvcLabels)
+
+					_, err = writers.Update(ctx, sdk, drd, pvc, emitEvents)
+					if err != nil {
+						return err
+					} else {
+						msg := fmt.Sprintf("unmarked pvc for deletion, removed labels %s and %s successfully in pvc [%s]", "toBeDeleted", "deletionTimestamp", pvc.GetName())
+						logger.Info(msg, "name", drd.Name, "namespace", drd.Namespace)
+					}
 				}
 			}
 		}

controllers/druid/util.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 )
 
 func firstNonEmptyStr(s1 string, s2 string) string {
@@ -77,3 +78,12 @@ func Str2Int(s string) int {
 	}
 	return i
 }
+
+// to find the time difference between two epoch times
+func timeDifference(epochTime1, epochTime2 int64) int64 {
+	t1 := time.Unix(epochTime1, 0)
+	t2 := time.Unix(epochTime2, 0)
+
+	diff := time.Duration(t2.Sub(t1))
+	return int64(diff.Seconds())
+}