Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ketch controller hang #260

Merged
merged 3 commits into from
Jun 15, 2022
Merged

Ketch controller hang #260

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d4c042
added events to workload struct to check for FailedCreate
kavinaravind Jun 8, 2022
3c72577
filtering events to fetch event attributed to workload
kavinaravind Jun 8, 2022
c6aa420
added timestamp event / workload rquality check for Pod
kavinaravind Jun 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions internal/controllers/app_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,12 @@ type condition struct {
Reason string
}

type eventCondition struct {
Type string
Reason string
Message string
}

// workload contains the needed information for watchDeployEvents logic
// deployments and statefulsets are both supported so it became necessary
// to abstract their common properties into a separate type
Expand All @@ -254,6 +260,7 @@ type workload struct {
Generation int
ObservedGeneration int
Conditions []condition
Events []eventCondition
}

type workloadClient struct {
Expand Down Expand Up @@ -284,6 +291,15 @@ func (cli workloadClient) Get(ctx context.Context) (*workload, error) {
for _, c := range o.Status.Conditions {
w.Conditions = append(w.Conditions, condition{Type: string(c.Type), Reason: c.Reason})
}
e, err := cli.k8sClient.CoreV1().Events(cli.workloadNamespace).List(ctx, metav1.ListOptions{FieldSelector: "involvedObject.name=" + o.Name, TypeMeta: metav1.TypeMeta{Kind: "Pod"}})
if err != nil {
return nil, err
}
for _, e := range e.Items {
if e.FirstTimestamp == o.ObjectMeta.CreationTimestamp {
w.Events = append(w.Events, eventCondition{Type: e.Type, Reason: e.Reason, Message: e.Message})
}
}
return &w, nil
case ketchv1.StatefulSetAppType:
o, err := cli.k8sClient.AppsV1().StatefulSets(cli.workloadNamespace).Get(ctx, cli.workloadName, metav1.GetOptions{})
Expand All @@ -303,6 +319,15 @@ func (cli workloadClient) Get(ctx context.Context) (*workload, error) {
for _, c := range o.Status.Conditions {
w.Conditions = append(w.Conditions, condition{Type: string(c.Type), Reason: c.Reason})
}
e, err := cli.k8sClient.CoreV1().Events(cli.workloadNamespace).List(ctx, metav1.ListOptions{FieldSelector: "involvedObject.name=" + o.Name, TypeMeta: metav1.TypeMeta{Kind: "StatefulSet"}})
if err != nil {
return nil, err
}
for _, e := range e.Items {
if e.FirstTimestamp == o.ObjectMeta.CreationTimestamp {
w.Events = append(w.Events, eventCondition{Type: e.Type, Reason: e.Reason, Message: e.Message})
}
}
return &w, nil
}
return nil, fmt.Errorf("unknown workload type")
Expand Down Expand Up @@ -495,6 +520,9 @@ func (r *AppReconciler) watchDeployEvents(ctx context.Context, app *ketchv1.App,
recorder.Eventf(app, v1.EventTypeWarning, ketchv1.AppReconcileError, "error getting deployments: %s", err.Error())
return err
}
if err := checkWorkloadEvent(wl); err != nil {
return err
}
select {
case <-time.After(100 * time.Millisecond):
case <-timeout:
Expand Down Expand Up @@ -681,6 +709,15 @@ func isDeploymentEvent(msg watch.Event, name string) bool {
return ok && strings.HasPrefix(evt.Name, name)
}

func checkWorkloadEvent(wl *workload) error {
for _, e := range wl.Events {
if e.Type == "Warning" && e.Reason == "FailedCreate" {
return errors.New(e.Message)
}
}
return nil
}

// createDeployTimeoutError gets pods that are not status == ready aggregates and returns the pod phase errors
func createDeployTimeoutError(ctx context.Context, cli kubernetes.Interface, app *ketchv1.App, timeout time.Duration, namespace, group, label string) error {
var deploymentVersion int
Expand Down