PB-4991: Update ApplicationBackupController to execute the post-exec …

…rule if they were not executed due to the backupCR deleteion
libopenstorage · Jan 4, 2024 · 307a89c · 307a89c
1 parent 6430214
commit 307a89c
Showing 1 changed file with 86 additions and 33 deletions.
diff --git a/pkg/applicationmanager/controllers/applicationbackup.go b/pkg/applicationmanager/controllers/applicationbackup.go
@@ -262,6 +262,46 @@ func (a *ApplicationBackupController) handle(ctx context.Context, backup *stork_
 			if !canDelete {
 				return nil
 			}
+			// run the post exec rules if the backup is in volume stage and execRulesCompleted is not completed
+			if _, ok := a.execRulesCompleted[string(backup.UID)]; !ok && backup.Status.Stage == stork_api.ApplicationBackupStageVolumes {
+				if terminationChannels, ok := a.terminationChannels[string(backup.UID)]; ok {
+					for _, channel := range terminationChannels {
+						logrus.Infof("Sending termination commands to kill pre-exec pod in non-kdmp driver path")
+						channel <- true
+					}
+				}
+				if backup.Spec.PostExecRule != "" {
+					log.ApplicationBackupLog(backup).Infof("Starting post-exec rule in delete path")
+					err = a.runPostExecRule(backup)
+					if err != nil {
+						message := fmt.Sprintf("Error running PostExecRule: %v", err)
+						log.ApplicationBackupLog(backup).Errorf(message)
+						a.recorder.Event(backup,
+							v1.EventTypeWarning,
+							string(stork_api.ApplicationBackupStatusFailed),
+							message)
+						return fmt.Errorf("%v", message)
+					}
+					a.execRulesCompleted[string(backup.UID)] = true
+				}
+			}
+
+			// Delete post-exec rule CR if any
+			if len(backup.Spec.PostExecRule) != 0 && backup.Annotations[utils.PxbackupAnnotationCreateByKey] == utils.PxbackupAnnotationCreateByValue && backup.Annotations["portworx.io/backup-by"] != "backup-schedule" && backup.Annotations["portworx.io/backup-by"] != "restore" {
+				log.ApplicationBackupLog(backup).Info("Delete post-exec rule CR")
+				//uid := backup.GetAnnotations()["portworx.io/backup-uid"]
+				//ruleCrName := getCrNameWithUID(backup.Spec.PostExecRule, uid)
+
+				err := storkops.Instance().DeleteRule(backup.Spec.PostExecRule, backup.Namespace)
+				log.ApplicationBackupLog(backup).Infof("Error value: %v", err)
+
+				if err != nil && !k8s_errors.IsNotFound(err) {
+					log.ApplicationBackupLog(backup).Infof("Error while deleting post exec rule: %v", err)
+					return err
+				}
+			}
+
+			// clean post exec rule cr only after execting the postexec rules or check if any auto-deletion of dependepent resources exists from k8s
 			// get-rid of map entry for termination channel and rule flag
 			if _, ok := a.terminationChannels[string(backup.UID)]; ok {
 				delete(a.terminationChannels, string(backup.UID))
@@ -572,8 +612,20 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio
 		backupStatusVolMap[statusVolume.Namespace+"-"+statusVolume.PersistentVolumeClaim] = ""
 	}
 
+	namespacedName := types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}
 	backup.Status.Stage = stork_api.ApplicationBackupStageVolumes
-	namespacedName := types.NamespacedName{}
+
+	backup, err = a.updateBackupCRInVolumeStage(
+		namespacedName,
+		stork_api.ApplicationBackupStatusInProgress,
+		backup.Status.Stage,
+		"Volume backups are in progress",
+		nil,
+	)
+	if err != nil {
+		return err
+	}
+
 	if IsVolsToBeBackedUp(backup) {
 		isResourceTypePVC := IsResourceTypePVC(backup)
 		objectMap := stork_api.CreateObjectsMap(backup.Spec.IncludeResources)
@@ -657,8 +709,6 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio
 			backup.Status.Volumes = make([]*stork_api.ApplicationBackupVolumeInfo, 0)
 		}
 
-		namespacedName.Namespace = backup.Namespace
-		namespacedName.Name = backup.Name
 		if len(backup.Status.Volumes) != pvcCount {
 			for driverName, pvcs := range pvcMappings {
 				var driver volume.Driver
@@ -885,38 +935,41 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio
 	driverCombo := a.checkVolumeDriverCombination(backup.Status.Volumes)
 	// If the driver combination of volumes onlykdmp or mixed of both kdmp and non-kdmp, call post exec rule
 	// backup of volume is success.
-	if driverCombo == kdmpDriverOnly || driverCombo == mixedDriver {
-		// Let's kill the pre-exec rule pod here so that application specific
-		// data  stream freezing logic works. Certain app actually unleash the WRITE when session ends.
-		// At this point we are dead sure that volume snapshot for all PVCs in the APP is done.. if not then
-		// there is issue...
-		// For detail refer pb-3823
-		for _, channel := range terminationChannels {
-			logrus.Infof("Sending termination commands to kill pre-exec pod in kdmp or mixed driver path")
-			channel <- true
-		}
-		//terminationChannels = nil
-		if backup.Spec.PostExecRule != "" {
-			log.ApplicationBackupLog(backup).Infof("Starting post-exec rule for kdmp and mixed driver path")
-			err = a.runPostExecRule(backup)
-			if err != nil {
-				message := fmt.Sprintf("Error running PostExecRule for kdmp and mixed driver scenario: %v", err)
-				log.ApplicationBackupLog(backup).Errorf(message)
-				a.recorder.Event(backup,
-					v1.EventTypeWarning,
-					string(stork_api.ApplicationBackupStatusFailed),
-					message)
-
-				backup.Status.Stage = stork_api.ApplicationBackupStageFinal
-				backup.Status.FinishTimestamp = metav1.Now()
-				backup.Status.LastUpdateTimestamp = metav1.Now()
-				backup.Status.Status = stork_api.ApplicationBackupStatusFailed
-				backup.Status.Reason = message
-				err = a.client.Update(context.TODO(), backup)
+	if !a.execRulesCompleted[string(backup.UID)] {
+		if driverCombo == kdmpDriverOnly || driverCombo == mixedDriver {
+			// Let's kill the pre-exec rule pod here so that application specific
+			// data  stream freezing logic works. Certain app actually unleash the WRITE when session ends.
+			// At this point we are dead sure that volume snapshot for all PVCs in the APP is done.. if not then
+			// there is issue...
+			// For detail refer pb-3823
+			for _, channel := range terminationChannels {
+				logrus.Infof("Sending termination commands to kill pre-exec pod in kdmp or mixed driver path")
+				channel <- true
+			}
+			//terminationChannels = nil
+			if backup.Spec.PostExecRule != "" {
+				log.ApplicationBackupLog(backup).Infof("Starting post-exec rule for kdmp and mixed driver path")
+				err = a.runPostExecRule(backup)
 				if err != nil {
-					return err
+					message := fmt.Sprintf("Error running PostExecRule for kdmp and mixed driver scenario: %v", err)
+					log.ApplicationBackupLog(backup).Errorf(message)
+					a.recorder.Event(backup,
+						v1.EventTypeWarning,
+						string(stork_api.ApplicationBackupStatusFailed),
+						message)
+
+					backup.Status.Stage = stork_api.ApplicationBackupStageFinal
+					backup.Status.FinishTimestamp = metav1.Now()
+					backup.Status.LastUpdateTimestamp = metav1.Now()
+					backup.Status.Status = stork_api.ApplicationBackupStatusFailed
+					backup.Status.Reason = message
+					err = a.client.Update(context.TODO(), backup)
+					if err != nil {
+						return err
+					}
+					return fmt.Errorf("%v", message)
 				}
-				return fmt.Errorf("%v", message)
+				a.execRulesCompleted[string(backup.UID)] = true
 			}
 		}
 	}