try to fix

kubeflow · Apr 17, 2020 · df464f2 · df464f2
1 parent 70d6013
commit df464f2
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 7 deletions.
diff --git a/pkg/controller.v1alpha3/experiment/experiment_controller.go b/pkg/controller.v1alpha3/experiment/experiment_controller.go
@@ -202,6 +202,9 @@ func (r *ReconcileExperiment) Reconcile(request reconcile.Request) (reconcile.Re
 				instance.MarkExperimentStatusRestarting(util.ExperimentRestartingReason, msg)
 			}
 		} else {
+			if instance.Spec.ResumePolicy != experimentsv1alpha3.LongRunning {
+				return r.terminateSuggestion(instance)
+			}
 			// If experiment is completed with no running trials, stop reconcile
 			if !instance.HasRunningTrials() {
 				return reconcile.Result{}, nil
@@ -262,9 +265,8 @@ func (r *ReconcileExperiment) ReconcileExperiment(instance *experimentsv1alpha3.
 	reconcileRequired := !instance.IsCompleted()
 	if reconcileRequired {
 		r.ReconcileTrials(instance, trials.Items)
-	} else if instance.Spec.ResumePolicy != experimentsv1alpha3.LongRunning {
-		return r.terminateSuggestion(instance)
 	}
+
 	return nil
 }
 

diff --git a/pkg/controller.v1alpha3/experiment/experiment_util.go b/pkg/controller.v1alpha3/experiment/experiment_util.go
@@ -112,7 +112,7 @@ func (r *ReconcileExperiment) updateFinalizers(instance *experimentsv1alpha3.Exp
 	}
 }
 
-func (r *ReconcileExperiment) terminateSuggestion(instance *experimentsv1alpha3.Experiment) error {
+func (r *ReconcileExperiment) terminateSuggestion(instance *experimentsv1alpha3.Experiment) (reconcile.Result, error) {
 	log.Info("Start terminating suggestion...")
 	suggestion := &suggestionsv1alpha3.Suggestion{}
 	err := r.Get(context.TODO(), types.NamespacedName{
@@ -121,16 +121,20 @@ func (r *ReconcileExperiment) terminateSuggestion(instance *experimentsv1alpha3.
 	}, suggestion)
 	if err != nil {
 		if errors.IsNotFound(err) {
-			return nil
+			return reconcile.Result{}, nil
 		}
-		return err
+		return reconcile.Result{}, err
 	}
 	if suggestion.IsCompleted() {
-		return nil
+		return reconcile.Result{}, nil
 	}
 	msg := "Suggestion is succeeded"
 	suggestion.MarkSuggestionStatusSucceeded(suggestionController.SuggestionSucceededReason, msg)
 	log.Info("Mark suggestion succeeded...")
 
-	return r.Client.Update(context.TODO(), suggestion)
+	if err := r.Update(context.TODO(), suggestion); err != nil {
+		return reconcile.Result{}, err
+	} else {
+		return reconcile.Result{Requeue: true}, nil
+	}
 }