Skip to content

Commit

Permalink
Use Patch operations to update AppWrapper.Status
Browse files Browse the repository at this point in the history 
Using Status().Patch(...) is preferred to Status().Update(...)
to reduce reconciler conflicts as Update on the subresource
still changes the resourceVersion on the root object.
  • Loading branch information
@dgrove-oss
dgrove-oss committed Aug 13, 2024
1 parent 147ec04 commit 310c209
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 42 deletions.
99 changes: 59 additions & 40 deletions internal/controller/appwrapper/appwrapper_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,14 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if !aw.DeletionTimestamp.IsZero() {
if controllerutil.ContainsFinalizer(aw, AppWrapperFinalizer) {
statusUpdated := false
orig := copyForStatusPatch(aw)
if meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)) {
if !r.deleteComponents(ctx, aw) {
// one or more components are still terminating
if aw.Status.Phase != workloadv1beta2.AppWrapperTerminating {
// Set Phase for better UX, but ignore errors. We still want to requeue after 5 seconds (not immediately)
aw.Status.Phase = workloadv1beta2.AppWrapperTerminating
_ = r.Status().Update(ctx, aw)
_ = r.Status().Patch(ctx, aw, client.MergeFrom(orig))
}
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil // check after a short while
}
Expand All @@ -136,7 +137,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
statusUpdated = true
}
if statusUpdated {
if err := r.Status().Update(ctx, aw); err != nil {
if err := r.Status().Patch(ctx, aw, client.MergeFrom(orig)); err != nil {
return ctrl.Result{}, err
}
}
Expand All @@ -160,18 +161,20 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
}

orig := copyForStatusPatch(aw)
if err := utils.EnsureComponentStatusInitialized(aw); err != nil {
return ctrl.Result{}, err
}

return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSuspended)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperSuspended)

case workloadv1beta2.AppWrapperSuspended: // no components deployed
if aw.Spec.Suspend {
return ctrl.Result{}, nil // remain suspended
}

// begin deployment
orig := copyForStatusPatch(aw)
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.QuotaReserved),
Status: metav1.ConditionTrue,
Expand All @@ -196,13 +199,14 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: string(workloadv1beta2.AppWrapperResuming),
Message: "Suspend is false",
})
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperResuming)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperResuming)

case workloadv1beta2.AppWrapperResuming: // deploying components
if aw.Spec.Suspend {
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSuspending) // abort deployment
return ctrl.Result{}, r.transitionToPhase(ctx, copyForStatusPatch(aw), aw, workloadv1beta2.AppWrapperSuspending) // abort deployment
}
err, fatal := r.createComponents(ctx, aw)
err, fatal := r.createComponents(ctx, aw) // NOTE: createComponents applies patches to aw.Status incrementally as resources are created
orig := copyForStatusPatch(aw)
if err != nil {
if !fatal {
startTime := meta.FindStatusCondition(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)).LastTransitionTime
Expand All @@ -221,16 +225,17 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
})
r.Recorder.Event(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), "CreateFailed: "+detailMsg)
if fatal {
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperFailed) // always move to failed on fatal error
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperFailed) // always move to failed on fatal error
} else {
return r.resetOrFail(ctx, aw, false, 1)
return ctrl.Result{}, r.resetOrFail(ctx, orig, aw, false, 1)
}
}
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperRunning)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperRunning)

case workloadv1beta2.AppWrapperRunning: // components deployed
orig := copyForStatusPatch(aw)
if aw.Spec.Suspend {
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSuspending) // begin undeployment
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperSuspending) // begin undeployment
}

// Gather status information at the Component and Pod level.
Expand All @@ -253,11 +258,11 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Message: detailMsg,
})
r.Recorder.Event(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), "MissingComponent: "+detailMsg)
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperFailed)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperFailed)
}

// If a component's controller has put it into a failed state, we do not need
// to allow any further grace period. The situation will not self-correct.
// to allow a grace period. The situation will not self-correct.
detailMsg = fmt.Sprintf("Found %v failed components", compStatus.failed)
if compStatus.failed > 0 {
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Expand All @@ -267,7 +272,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Message: detailMsg,
})
r.Recorder.Event(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), "FailedComponent: "+detailMsg)
return r.resetOrFail(ctx, aw, podStatus.terminalFailure, 1)
return ctrl.Result{}, r.resetOrFail(ctx, orig, aw, podStatus.terminalFailure, 1)
}

// Handle Success
Expand All @@ -285,7 +290,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: string(workloadv1beta2.AppWrapperSucceeded),
Message: msg,
})
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSucceeded)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperSucceeded)
}

// Handle Failed Pods
Expand All @@ -303,10 +308,10 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
now := time.Now()
deadline := whenDetected.Add(gracePeriod)
if now.Before(deadline) {
return requeueAfter(deadline.Sub(now), r.Status().Update(ctx, aw))
return requeueAfter(deadline.Sub(now), r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
} else {
r.Recorder.Eventf(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), "FoundFailedPods: %v failed pods", podStatus.failed)
return r.resetOrFail(ctx, aw, podStatus.terminalFailure, 1)
return ctrl.Result{}, r.resetOrFail(ctx, orig, aw, podStatus.terminalFailure, 1)
}
}

Expand All @@ -320,7 +325,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Message: detailMsg,
})
r.Recorder.Event(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), detailMsg)
return r.resetOrFail(ctx, aw, false, 0) // Autopilot triggered evacuation does not increment retry count
return ctrl.Result{}, r.resetOrFail(ctx, orig, aw, false, 0) // Autopilot triggered evacuation does not increment retry count
}

clearCondition(aw, workloadv1beta2.Unhealthy, "FoundNoFailedPods", "")
Expand All @@ -332,7 +337,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: "SufficientPodsReady",
Message: fmt.Sprintf("%v pods running; %v pods succeeded", podStatus.running, podStatus.succeeded),
})
return requeueAfter(time.Minute, r.Status().Update(ctx, aw))
return requeueAfter(time.Minute, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}

// Not ready yet; either continue to wait or giveup if the warmup period has expired
Expand All @@ -346,7 +351,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
graceDuration = r.admissionGraceDuration(ctx, aw)
}
if time.Now().Before(whenDeployed.Add(graceDuration)) {
return requeueAfter(5*time.Second, r.Status().Update(ctx, aw))
return requeueAfter(5*time.Second, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
} else {
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.Unhealthy),
Expand All @@ -355,14 +360,15 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Message: podDetailsMessage,
})
r.Recorder.Event(aw, v1.EventTypeNormal, string(workloadv1beta2.Unhealthy), "InsufficientPodsReady: "+podDetailsMessage)
return r.resetOrFail(ctx, aw, podStatus.terminalFailure, 1)
return ctrl.Result{}, r.resetOrFail(ctx, orig, aw, podStatus.terminalFailure, 1)
}

case workloadv1beta2.AppWrapperSuspending: // undeploying components
orig := copyForStatusPatch(aw)
// finish undeploying components irrespective of desired state (suspend bit)
if meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)) {
if !r.deleteComponents(ctx, aw) {
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
return requeueAfter(5*time.Second, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.ResourcesDeployed),
Expand All @@ -379,17 +385,18 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
})
clearCondition(aw, workloadv1beta2.PodsReady, string(workloadv1beta2.AppWrapperSuspended), "")
clearCondition(aw, workloadv1beta2.Unhealthy, string(workloadv1beta2.AppWrapperSuspended), "")
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSuspended)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperSuspended)

case workloadv1beta2.AppWrapperResetting:
orig := copyForStatusPatch(aw)
if aw.Spec.Suspend {
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperSuspending) // Suspending trumps Resetting
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperSuspending) // Suspending trumps Resetting
}

clearCondition(aw, workloadv1beta2.PodsReady, string(workloadv1beta2.AppWrapperResetting), "")
if meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)) {
if !r.deleteComponents(ctx, aw) {
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
return requeueAfter(5*time.Second, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.ResourcesDeployed),
Expand All @@ -405,7 +412,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
now := time.Now()
deadline := whenReset.Add(pauseDuration)
if now.Before(deadline) {
return requeueAfter(deadline.Sub(now), r.Status().Update(ctx, aw))
return requeueAfter(deadline.Sub(now), r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}

meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Expand All @@ -414,7 +421,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: string(workloadv1beta2.AppWrapperResuming),
Message: "Reset complete; resuming",
})
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperResuming)
return ctrl.Result{}, r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperResuming)

case workloadv1beta2.AppWrapperFailed:
// Support for debugging failed jobs.
Expand All @@ -423,6 +430,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// a failed appwrapper unless Kueue preempts it by setting Suspend to true.
deletionDelay := r.deletionOnFailureGraceDuration(ctx, aw)

orig := copyForStatusPatch(aw)
if deletionDelay > 0 && !aw.Spec.Suspend {
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.DeletingResources),
Expand All @@ -435,13 +443,13 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
now := time.Now()
deadline := whenDelayed.Add(deletionDelay)
if now.Before(deadline) {
return requeueAfter(deadline.Sub(now), r.Status().Update(ctx, aw))
return requeueAfter(deadline.Sub(now), r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}
}

if meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)) {
if !r.deleteComponents(ctx, aw) {
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
return requeueAfter(5*time.Second, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}
msg := "Resources deleted for failed AppWrapper"
if deletionDelay > 0 && aw.Spec.Suspend {
Expand All @@ -460,7 +468,7 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: string(workloadv1beta2.AppWrapperFailed),
Message: "No resources deployed",
})
return ctrl.Result{}, r.Status().Update(ctx, aw)
return ctrl.Result{}, r.Status().Patch(ctx, aw, client.MergeFrom(orig))

case workloadv1beta2.AppWrapperSucceeded:
if meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed)) {
Expand All @@ -469,42 +477,43 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
now := time.Now()
deadline := whenSucceeded.Add(deletionDelay)
if now.Before(deadline) {
return requeueAfter(deadline.Sub(now), r.Status().Update(ctx, aw))
return requeueAfter(deadline.Sub(now), nil)
}

orig := copyForStatusPatch(aw)
if !r.deleteComponents(ctx, aw) {
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
return requeueAfter(5*time.Second, r.Status().Patch(ctx, aw, client.MergeFrom(orig)))
}
meta.SetStatusCondition(&aw.Status.Conditions, metav1.Condition{
Type: string(workloadv1beta2.ResourcesDeployed),
Status: metav1.ConditionFalse,
Reason: string(workloadv1beta2.AppWrapperSucceeded),
Message: fmt.Sprintf("Time to live after success of %v expired", deletionDelay),
})
return ctrl.Result{}, r.Status().Update(ctx, aw)
return ctrl.Result{}, r.Status().Patch(ctx, aw, client.MergeFrom(orig))
}
return ctrl.Result{}, nil
}

return ctrl.Result{}, nil
}

func (r *AppWrapperReconciler) updateStatus(ctx context.Context, aw *workloadv1beta2.AppWrapper, phase workloadv1beta2.AppWrapperPhase) (ctrl.Result, error) {
aw.Status.Phase = phase
if err := r.Status().Update(ctx, aw); err != nil {
return ctrl.Result{}, err
func (r *AppWrapperReconciler) transitionToPhase(ctx context.Context, orig *workloadv1beta2.AppWrapper, modified *workloadv1beta2.AppWrapper, phase workloadv1beta2.AppWrapperPhase) error {
modified.Status.Phase = phase
if err := r.Status().Patch(ctx, modified, client.MergeFrom(orig)); err != nil {
return err
}
log.FromContext(ctx).Info(string(phase), "phase", phase)
return ctrl.Result{}, nil
return nil
}

func (r *AppWrapperReconciler) resetOrFail(ctx context.Context, aw *workloadv1beta2.AppWrapper, terminalFailure bool, retryIncrement int32) (ctrl.Result, error) {
func (r *AppWrapperReconciler) resetOrFail(ctx context.Context, orig *workloadv1beta2.AppWrapper, aw *workloadv1beta2.AppWrapper, terminalFailure bool, retryIncrement int32) error {
maxRetries := r.retryLimit(ctx, aw)
if !terminalFailure && aw.Status.Retries < maxRetries {
aw.Status.Retries += retryIncrement
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperResetting)
return r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperResetting)
} else {
return r.updateStatus(ctx, aw, workloadv1beta2.AppWrapperFailed)
return r.transitionToPhase(ctx, orig, aw, workloadv1beta2.AppWrapperFailed)
}
}

Expand Down Expand Up @@ -897,6 +906,16 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}

// copyForStatusPatch returns an AppWrapper with an empty Spec and a DeepCopy of orig's Status for use in a subsequent Status().Patch(...) call
func copyForStatusPatch(orig *workloadv1beta2.AppWrapper) *workloadv1beta2.AppWrapper {
copy := workloadv1beta2.AppWrapper{
TypeMeta: orig.TypeMeta,
ObjectMeta: orig.ObjectMeta,
Status: *orig.Status.DeepCopy(),
}
return &copy
}

// requeueAfter requeues the request after the specified duration
func requeueAfter(duration time.Duration, err error) (ctrl.Result, error) {
if err != nil {
Expand Down
7 changes: 5 additions & 2 deletions internal/controller/appwrapper/resource_management.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ func (r *AppWrapperReconciler) createComponent(ctx context.Context, aw *workload
return err, true
}

orig := copyForStatusPatch(aw)
if meta.FindStatusCondition(aw.Status.ComponentStatus[componentIdx].Conditions, string(workloadv1beta2.ResourcesDeployed)) == nil {
aw.Status.ComponentStatus[componentIdx].Name = obj.GetName()
aw.Status.ComponentStatus[componentIdx].Kind = obj.GetKind()
Expand All @@ -288,7 +289,7 @@ func (r *AppWrapperReconciler) createComponent(ctx context.Context, aw *workload
Status: metav1.ConditionUnknown,
Reason: "ComponentCreationInitiated",
})
if err := r.Status().Update(ctx, aw); err != nil {
if err := r.Status().Patch(ctx, aw, client.MergeFrom(orig)); err != nil {
return err, false
}
}
Expand All @@ -309,19 +310,21 @@ func (r *AppWrapperReconciler) createComponent(ctx context.Context, aw *workload
}
}

orig = copyForStatusPatch(aw)
aw.Status.ComponentStatus[componentIdx].Name = obj.GetName() // Update name to support usage of GenerateName
meta.SetStatusCondition(&aw.Status.ComponentStatus[componentIdx].Conditions, metav1.Condition{
Type: string(workloadv1beta2.ResourcesDeployed),
Status: metav1.ConditionTrue,
Reason: "ComponentCreatedSuccessfully",
})
if err := r.Status().Update(ctx, aw); err != nil {
if err := r.Status().Patch(ctx, aw, client.MergeFrom(orig)); err != nil {
return err, false
}

return nil, false
}

// createComponents incrementally patches aw.Status -- MUST NOT CARRY STATUS PATCHES ACROSS INVOCATIONS
func (r *AppWrapperReconciler) createComponents(ctx context.Context, aw *workloadv1beta2.AppWrapper) (error, bool) {
for componentIdx := range aw.Spec.Components {
if !meta.IsStatusConditionTrue(aw.Status.ComponentStatus[componentIdx].Conditions, string(workloadv1beta2.ResourcesDeployed)) {
Expand Down

0 comments on commit 310c209

Please sign in to comment.