From 048271c7ef3597dd1e16a1923d4b839d1d97da9d Mon Sep 17 00:00:00 2001 From: Santosh Kumar Gajawada Date: Thu, 4 Jan 2024 06:40:08 +0530 Subject: [PATCH] PB-4991: Update ApplicationBackupController to execute the post-exec rule if they were not executed due to the backupCR deleteion --- .../controllers/applicationbackup.go | 119 +++++++++++++----- 1 file changed, 86 insertions(+), 33 deletions(-) diff --git a/pkg/applicationmanager/controllers/applicationbackup.go b/pkg/applicationmanager/controllers/applicationbackup.go index 30fd47cec4..26eb6e2dbe 100644 --- a/pkg/applicationmanager/controllers/applicationbackup.go +++ b/pkg/applicationmanager/controllers/applicationbackup.go @@ -262,6 +262,46 @@ func (a *ApplicationBackupController) handle(ctx context.Context, backup *stork_ if !canDelete { return nil } + // run the post exec rules if the backup is in volume stage and execRulesCompleted is not completed + if _, ok := a.execRulesCompleted[string(backup.UID)]; !ok && backup.Status.Stage == stork_api.ApplicationBackupStageVolumes { + if terminationChannels, ok := a.terminationChannels[string(backup.UID)]; ok { + for _, channel := range terminationChannels { + logrus.Infof("Sending termination commands to kill pre-exec pod in non-kdmp driver path") + channel <- true + } + } + if backup.Spec.PostExecRule != "" { + log.ApplicationBackupLog(backup).Infof("Starting post-exec rule in delete path") + err = a.runPostExecRule(backup) + if err != nil { + message := fmt.Sprintf("Error running PostExecRule: %v", err) + log.ApplicationBackupLog(backup).Errorf(message) + a.recorder.Event(backup, + v1.EventTypeWarning, + string(stork_api.ApplicationBackupStatusFailed), + message) + return fmt.Errorf("%v", message) + } + a.execRulesCompleted[string(backup.UID)] = true + } + } + + // Delete post-exec rule CR if any + if len(backup.Spec.PostExecRule) != 0 && backup.Annotations[utils.PxbackupAnnotationCreateByKey] == utils.PxbackupAnnotationCreateByValue && backup.Annotations["portworx.io/backup-by"] != "backup-schedule" && backup.Annotations["portworx.io/backup-by"] != "restore" { + log.ApplicationBackupLog(backup).Info("Delete post-exec rule CR") + //uid := backup.GetAnnotations()["portworx.io/backup-uid"] + //ruleCrName := getCrNameWithUID(backup.Spec.PostExecRule, uid) + + err := storkops.Instance().DeleteRule(backup.Spec.PostExecRule, backup.Namespace) + log.ApplicationBackupLog(backup).Infof("Error value: %v", err) + + if err != nil && !k8s_errors.IsNotFound(err) { + log.ApplicationBackupLog(backup).Infof("Error while deleting post exec rule: %v", err) + return err + } + } + + // clean post exec rule cr only after execting the postexec rules or check if any auto-deletion of dependepent resources exists from k8s // get-rid of map entry for termination channel and rule flag if _, ok := a.terminationChannels[string(backup.UID)]; ok { delete(a.terminationChannels, string(backup.UID)) @@ -572,8 +612,20 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio backupStatusVolMap[statusVolume.Namespace+"-"+statusVolume.PersistentVolumeClaim] = "" } + namespacedName := types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name} backup.Status.Stage = stork_api.ApplicationBackupStageVolumes - namespacedName := types.NamespacedName{} + + backup, err = a.updateBackupCRInVolumeStage( + namespacedName, + stork_api.ApplicationBackupStatusInProgress, + backup.Status.Stage, + "Volume backups are in progress", + nil, + ) + if err != nil { + return err + } + if IsVolsToBeBackedUp(backup) { isResourceTypePVC := IsResourceTypePVC(backup) objectMap := stork_api.CreateObjectsMap(backup.Spec.IncludeResources) @@ -657,8 +709,6 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio backup.Status.Volumes = make([]*stork_api.ApplicationBackupVolumeInfo, 0) } - namespacedName.Namespace = backup.Namespace - namespacedName.Name = backup.Name if len(backup.Status.Volumes) != pvcCount { for driverName, pvcs := range pvcMappings { var driver volume.Driver @@ -885,38 +935,41 @@ func (a *ApplicationBackupController) backupVolumes(backup *stork_api.Applicatio driverCombo := a.checkVolumeDriverCombination(backup.Status.Volumes) // If the driver combination of volumes onlykdmp or mixed of both kdmp and non-kdmp, call post exec rule // backup of volume is success. - if driverCombo == kdmpDriverOnly || driverCombo == mixedDriver { - // Let's kill the pre-exec rule pod here so that application specific - // data stream freezing logic works. Certain app actually unleash the WRITE when session ends. - // At this point we are dead sure that volume snapshot for all PVCs in the APP is done.. if not then - // there is issue... - // For detail refer pb-3823 - for _, channel := range terminationChannels { - logrus.Infof("Sending termination commands to kill pre-exec pod in kdmp or mixed driver path") - channel <- true - } - //terminationChannels = nil - if backup.Spec.PostExecRule != "" { - log.ApplicationBackupLog(backup).Infof("Starting post-exec rule for kdmp and mixed driver path") - err = a.runPostExecRule(backup) - if err != nil { - message := fmt.Sprintf("Error running PostExecRule for kdmp and mixed driver scenario: %v", err) - log.ApplicationBackupLog(backup).Errorf(message) - a.recorder.Event(backup, - v1.EventTypeWarning, - string(stork_api.ApplicationBackupStatusFailed), - message) - - backup.Status.Stage = stork_api.ApplicationBackupStageFinal - backup.Status.FinishTimestamp = metav1.Now() - backup.Status.LastUpdateTimestamp = metav1.Now() - backup.Status.Status = stork_api.ApplicationBackupStatusFailed - backup.Status.Reason = message - err = a.client.Update(context.TODO(), backup) + if !a.execRulesCompleted[string(backup.UID)] { + if driverCombo == kdmpDriverOnly || driverCombo == mixedDriver { + // Let's kill the pre-exec rule pod here so that application specific + // data stream freezing logic works. Certain app actually unleash the WRITE when session ends. + // At this point we are dead sure that volume snapshot for all PVCs in the APP is done.. if not then + // there is issue... + // For detail refer pb-3823 + for _, channel := range terminationChannels { + logrus.Infof("Sending termination commands to kill pre-exec pod in kdmp or mixed driver path") + channel <- true + } + //terminationChannels = nil + if backup.Spec.PostExecRule != "" { + log.ApplicationBackupLog(backup).Infof("Starting post-exec rule for kdmp and mixed driver path") + err = a.runPostExecRule(backup) if err != nil { - return err + message := fmt.Sprintf("Error running PostExecRule for kdmp and mixed driver scenario: %v", err) + log.ApplicationBackupLog(backup).Errorf(message) + a.recorder.Event(backup, + v1.EventTypeWarning, + string(stork_api.ApplicationBackupStatusFailed), + message) + + backup.Status.Stage = stork_api.ApplicationBackupStageFinal + backup.Status.FinishTimestamp = metav1.Now() + backup.Status.LastUpdateTimestamp = metav1.Now() + backup.Status.Status = stork_api.ApplicationBackupStatusFailed + backup.Status.Reason = message + err = a.client.Update(context.TODO(), backup) + if err != nil { + return err + } + return fmt.Errorf("%v", message) } - return fmt.Errorf("%v", message) + a.execRulesCompleted[string(backup.UID)] = true } } }