Skip to content

Commit

Permalink
Disable component-level failure detection for Ray (#174) (#177)
Browse files Browse the repository at this point in the history
In KubeRay 1.1, status.state == failed is not a stable terminal state,
therefore we cannot treat it as a signal to initiate a resetOrFail
operation on the AppWrapper.
  • Loading branch information
dgrove-oss authored Jun 27, 2024
1 parent 5a3de24 commit aebdb4d
Showing 1 changed file with 9 additions and 0 deletions.
9 changes: 9 additions & 0 deletions internal/controller/appwrapper/appwrapper_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,10 @@ func (r *AppWrapperReconciler) getComponentStatus(ctx context.Context, aw *workl
if obj.GetDeletionTimestamp().IsZero() {
summary.deployed += 1

/* Disabled because failed is not a terminal state.
* We've observed RC transiently entering "failed" before becoming "ready" due to ingress not being ready
* TODO: Explore fixing in upstream projects.
// RayCluster is failed if status.State is "failed"
status, ok := obj.UnstructuredContent()["status"]
if !ok {
Expand All @@ -595,6 +599,7 @@ func (r *AppWrapperReconciler) getComponentStatus(ctx context.Context, aw *workl
if state.(string) == "failed" {
summary.failed += 1
}
*/
}
} else if !apierrors.IsNotFound(err) {
return nil, err
Expand All @@ -608,6 +613,9 @@ func (r *AppWrapperReconciler) getComponentStatus(ctx context.Context, aw *workl
if obj.GetDeletionTimestamp().IsZero() {
summary.deployed += 1

/* Disabled because we are not sure if failed is a terminal state.
* TODO: Determine whether or not RayJob has the same issue as RayCluster
// RayJob is failed if status.jobsStatus is "FAILED"
status, ok := obj.UnstructuredContent()["status"]
if !ok {
Expand All @@ -620,6 +628,7 @@ func (r *AppWrapperReconciler) getComponentStatus(ctx context.Context, aw *workl
if jobStatus.(string) == "FAILED" {
summary.failed += 1
}
*/
}
} else if !apierrors.IsNotFound(err) {
return nil, err
Expand Down

0 comments on commit aebdb4d

Please sign in to comment.