diff --git a/internal/backoff/backoff.go b/internal/backoff/backoff.go index a74d2e3..a851519 100644 --- a/internal/backoff/backoff.go +++ b/internal/backoff/backoff.go @@ -22,13 +22,16 @@ type Backoff struct { func NewBackoff(maxDelay time.Duration) *Backoff { return &Backoff{ activities: make(map[any]any), - // resulting per-item backoff is the maximum of a 300-times-20ms-then-maxDelay per-item limiter, - // and an overall 10-per-second-burst-20 bucket limiter; - // as a consequence, we have up to 20 almost immediate retries, then a phase of 10 retries per seconnd - // for approximately 30s, and then slow retries at the rate given by maxDelay + // resulting per-item backoff is the maximum of a 200-times-50ms-then-maxDelay per-item limiter, + // and an overall 5-per-second-burst-20 bucket limiter; + // as a consequence, we have up to + // - up to 20 almost immediate retries + // - then then a phase of 5 guaranteed retries per seconnd (could be more if burst capacity is refilled + // because of the duration of the reconcile logic execution itself) + // - finally (after 200 iterations) slow retries at the rate given by maxDelay limiter: workqueue.NewMaxOfRateLimiter( - workqueue.NewItemFastSlowRateLimiter(20*time.Millisecond, maxDelay, 300), - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 20)}, + workqueue.NewItemFastSlowRateLimiter(50*time.Millisecond, maxDelay, 200), + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(5), 20)}, ), } } diff --git a/pkg/component/component.go b/pkg/component/component.go index c75f765..853c43b 100644 --- a/pkg/component/component.go +++ b/pkg/component/component.go @@ -11,8 +11,6 @@ import ( "reflect" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/sap/component-operator-runtime/internal/walk" ) @@ -88,7 +86,18 @@ func assertRetryConfiguration[T Component](component T) (RetryConfiguration, boo return nil, false } -// Calculate digest of given component, honoring annotations, spec, and references +// Check if given component or its spec implements TimeoutConfiguration (and return it). +func assertTimeoutConfiguration[T Component](component T) (TimeoutConfiguration, bool) { + if timeoutConfiguration, ok := Component(component).(TimeoutConfiguration); ok { + return timeoutConfiguration, true + } + if timeoutConfiguration, ok := getSpec(component).(TimeoutConfiguration); ok { + return timeoutConfiguration, true + } + return nil, false +} + +// Calculate digest of given component, honoring annotations, spec, and references. func calculateComponentDigest[T Component](component T) string { digestData := make(map[string]any) spec := getSpec(component) @@ -120,8 +129,7 @@ func calculateComponentDigest[T Component](component T) string { // note: this panic is ok because walk.Walk() only produces errors if the given walker function raises any (which ours here does not do) panic("this cannot happen") } - // note: this must() is ok because digestData should contain only serializable stuff - return sha256hex(must(json.Marshal(digestData))) + return calculateDigest(digestData) } // Implement the PlacementConfiguration interface. @@ -178,46 +186,62 @@ func (s *Status) IsReady() bool { return s.State == StateReady } -// Get state (and related details). -func (s *Status) GetState() (State, string, string) { - var cond *Condition +// Implement the TimeoutConfiguration interface. +func (s *TimeoutSpec) GetTimeout() time.Duration { + if s.Timeout != nil { + return s.Timeout.Duration + } + return time.Duration(0) +} + +// Get condition (and return nil if not existing). +// Caveat: the returned pointer might become invalid if further appends happen to the Conditions slice in the status object. +func (s *Status) getCondition(condType ConditionType) *Condition { for i := 0; i < len(s.Conditions); i++ { - if s.Conditions[i].Type == ConditionTypeReady { - cond = &s.Conditions[i] - break + if s.Conditions[i].Type == condType { + return &s.Conditions[i] } } - if cond == nil { - return s.State, "", "" - } - return s.State, cond.Reason, cond.Message + return nil } -// Set state and ready condition in status (according to the state value provided), -func (s *Status) SetState(state State, reason string, message string) { +// Get condition (adding it with initial values if not existing). +// Caveat: the returned pointer might become invalid if further appends happen to the Conditions slice in the status object. +func (s *Status) getOrAddCondition(condType ConditionType) *Condition { var cond *Condition for i := 0; i < len(s.Conditions); i++ { - if s.Conditions[i].Type == ConditionTypeReady { + if s.Conditions[i].Type == condType { cond = &s.Conditions[i] break } } if cond == nil { - s.Conditions = append(s.Conditions, Condition{Type: ConditionTypeReady}) + s.Conditions = append(s.Conditions, Condition{Type: condType, Status: ConditionUnknown}) cond = &s.Conditions[len(s.Conditions)-1] } - var status ConditionStatus + return cond +} + +// Get state (and related details). +func (s *Status) GetState() (State, string, string) { + cond := s.getCondition(ConditionTypeReady) + if cond == nil { + return s.State, "", "" + } + return s.State, cond.Reason, cond.Message +} + +// Set state and ready condition in status (according to the state value provided). +// Note: this method does not touch the condition's LastTransitionTime. +func (s *Status) SetState(state State, reason string, message string) { + cond := s.getOrAddCondition(ConditionTypeReady) switch state { case StateReady: - status = ConditionTrue + cond.Status = ConditionTrue case StateError: - status = ConditionFalse + cond.Status = ConditionFalse default: - status = ConditionUnknown - } - if status != cond.Status { - cond.Status = status - cond.LastTransitionTime = ref(metav1.Now()) + cond.Status = ConditionUnknown } cond.Reason = reason cond.Message = message diff --git a/pkg/component/reconciler.go b/pkg/component/reconciler.go index 36a3c40..b1c8503 100644 --- a/pkg/component/reconciler.go +++ b/pkg/component/reconciler.go @@ -61,6 +61,7 @@ const ( readyConditionReasonProcessing = "Processing" readyConditionReasonReady = "Ready" readyConditionReasonError = "Error" + readyConditionReasonTimeout = "Timeout" readyConditionReasonDeletionPending = "DeletionPending" readyConditionReasonDeletionBlocked = "DeletionBlocked" readyConditionReasonDeletionProcessing = "DeletionProcessing" @@ -169,11 +170,7 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result } component.GetObjectKind().SetGroupVersionKind(r.groupVersionKind) - // convenience accessors - status := component.GetStatus() - savedStatus := status.DeepCopy() - - // requeue/retry interval + // fetch requeue interval, retry interval and timeout requeueInterval := time.Duration(0) if requeueConfiguration, ok := assertRequeueConfiguration(component); ok { requeueInterval = requeueConfiguration.GetRequeueInterval() @@ -188,6 +185,17 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result if retryInterval == 0 { retryInterval = requeueInterval } + timeout := time.Duration(0) + if timeoutConfiguration, ok := assertTimeoutConfiguration(component); ok { + timeout = timeoutConfiguration.GetTimeout() + } + if timeout == 0 { + timeout = requeueInterval + } + + // convenience accessors + status := component.GetStatus() + savedStatus := status.DeepCopy() // always attempt to update the status skipStatusUpdate := false @@ -197,11 +205,27 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result // re-panic in order skip the remaining steps panic(r) } + + status.ObservedGeneration = component.GetGeneration() + if status.State == StateReady || err != nil { + // clear backoff if state is ready (obviously) or if there is an error; + // even is the error is a RetriableError which will be turned into a non-error; + // this is correct, because in that case, the RequeueAfter will be determined through the RetriableError r.backoff.Forget(req) } - status.ObservedGeneration = component.GetGeneration() + if status.State != StateProcessing || err != nil { + // clear ProcessingDigest and ProcessingSince in all non-error cases where state is StateProcessing + status.ProcessingDigest = "" + status.ProcessingSince = nil + } + if status.State == StateProcessing && now.Sub(status.ProcessingSince.Time) >= timeout { + // TODO: maybe it would be better to have a dedicated StateTimeout? + status.SetState(StateError, readyConditionReasonTimeout, "Reconcilation of dependent resources timed out") + } + if err != nil { + // convert retriable errors into non-errors (Pending or DeletionPending state), and return specified or default backoff retriableError := &types.RetriableError{} if errors.As(err, retriableError) { retryAfter := retriableError.RetryAfter() @@ -220,10 +244,12 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result status.SetState(StateError, readyConditionReasonError, err.Error()) } } + if result.RequeueAfter > 0 { // add jitter of 1-5 percent to RequeueAfter addJitter(&result.RequeueAfter, 1, 5) } + log.V(1).Info("reconcile done", "withError", err != nil, "requeue", result.Requeue || result.RequeueAfter > 0, "requeueAfter", result.RequeueAfter.String()) if err != nil { if status, ok := err.(apierrors.APIStatus); ok || errors.As(err, &status) { @@ -232,22 +258,34 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result metrics.ReconcileErrors.WithLabelValues(r.controllerName, "other").Inc() } } - // TODO: should we move this behind the DeepEqual check below? - // note: it seems that no events will be written if the component's namespace is in deletion + + // TODO: should we move this behind the DeepEqual check below to avoid noise? + // also note: it seems that no events will be written if the component's namespace is in deletion state, reason, message := status.GetState() if state == StateError { r.client.EventRecorder().Event(component, corev1.EventTypeWarning, reason, message) } else { r.client.EventRecorder().Event(component, corev1.EventTypeNormal, reason, message) } + if skipStatusUpdate { return } if reflect.DeepEqual(status, savedStatus) { return } - // note: it's crucial to set the following timestamp late (otherwise the DeepEqual() check before would always be false) + + // note: it's crucial to set the following timestamps late (otherwise the DeepEqual() check above would always be false) + // on the other hand it's a bit weird, because LastObservedAt will not be updated if no other changes have happened to the status; + // and same for the conditions' LastTransitionTime timestamps; + // maybe we should remove this optimization, and always do the Update() call status.LastObservedAt = &now + for i := 0; i < len(status.Conditions); i++ { + cond := &status.Conditions[i] + if savedCond := savedStatus.getCondition(cond.Type); savedCond == nil || cond.Status != savedCond.Status { + cond.LastTransitionTime = &now + } + } if updateErr := r.client.Status().Update(ctx, component, client.FieldOwner(r.name)); updateErr != nil { err = utilerrors.NewAggregate([]error{err, updateErr}) result = ctrl.Result{} @@ -256,7 +294,7 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result // set a first status (and requeue, because the status update itself will not trigger another reconciliation because of the event filter set) if status.ObservedGeneration <= 0 { - status.SetState(StateProcessing, readyConditionReasonNew, "First seen") + status.SetState(StatePending, readyConditionReasonNew, "First seen") return ctrl.Result{Requeue: true}, nil } @@ -301,7 +339,8 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result return ctrl.Result{}, errors.Wrap(err, "error adding finalizer") } // trigger another round trip - // this is necessary because the update call invalidates potential changes done by the post-read hook above + // this is necessary because the update call invalidates potential changes done to the component by the post-read + // hook above; this means, not to the object itself, but for example to loaded secrets or config maps; // in the following round trip, the finalizer will already be there, and the update will not happen again return ctrl.Result{Requeue: true}, nil } @@ -312,7 +351,7 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result return ctrl.Result{}, errors.Wrapf(err, "error running pre-reconcile hook (%d)", hookOrder) } } - ok, err := target.Apply(ctx, component) + ok, digest, err := target.Apply(ctx, component) if err != nil { log.V(1).Info("error while reconciling dependent resources") return ctrl.Result{}, errors.Wrap(err, "error reconciling dependent resources") @@ -324,16 +363,21 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result } } log.V(1).Info("all dependent resources successfully reconciled") - status.SetState(StateReady, readyConditionReasonReady, "Dependent resources successfully reconciled") status.AppliedGeneration = component.GetGeneration() status.LastAppliedAt = &now + status.SetState(StateReady, readyConditionReasonReady, "Dependent resources successfully reconciled") return ctrl.Result{RequeueAfter: requeueInterval}, nil } else { log.V(1).Info("not all dependent resources successfully reconciled") - status.SetState(StateProcessing, readyConditionReasonProcessing, "Reconcilation of dependent resources triggered; waiting until all dependent resources are ready") + if digest != status.ProcessingDigest { + status.ProcessingDigest = digest + status.ProcessingSince = &now + r.backoff.Forget(req) + } if !reflect.DeepEqual(status.Inventory, savedStatus.Inventory) { r.backoff.Forget(req) } + status.SetState(StateProcessing, readyConditionReasonProcessing, "Reconcilation of dependent resources triggered; waiting until all dependent resources are ready") return ctrl.Result{RequeueAfter: r.backoff.Next(req, readyConditionReasonProcessing)}, nil } } else { @@ -352,16 +396,16 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result log.V(1).Info("deletion not allowed") // TODO: have an additional StateDeletionBlocked? // TODO: eliminate this msg logic - status.SetState(StateDeleting, readyConditionReasonDeletionBlocked, "Deletion blocked: "+msg) r.client.EventRecorder().Event(component, corev1.EventTypeNormal, readyConditionReasonDeletionBlocked, "Deletion blocked: "+msg) + status.SetState(StateDeleting, readyConditionReasonDeletionBlocked, "Deletion blocked: "+msg) return ctrl.Result{RequeueAfter: 1*time.Second + r.backoff.Next(req, readyConditionReasonDeletionBlocked)}, nil } if len(slices.Remove(component.GetFinalizers(), r.name)) > 0 { // deletion is blocked because of foreign finalizers log.V(1).Info("deleted blocked due to existence of foreign finalizers") // TODO: have an additional StateDeletionBlocked? - status.SetState(StateDeleting, readyConditionReasonDeletionBlocked, "Deletion blocked due to existing foreign finalizers") r.client.EventRecorder().Event(component, corev1.EventTypeNormal, readyConditionReasonDeletionBlocked, "Deletion blocked due to existing foreign finalizers") + status.SetState(StateDeleting, readyConditionReasonDeletionBlocked, "Deletion blocked due to existing foreign finalizers") return ctrl.Result{RequeueAfter: 1*time.Second + r.backoff.Next(req, readyConditionReasonDeletionBlocked)}, nil } // deletion case @@ -392,10 +436,10 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result } else { // deletion triggered for dependent resources, but some are not yet gone log.V(1).Info("not all dependent resources are successfully deleted") - status.SetState(StateDeleting, readyConditionReasonDeletionProcessing, "Deletion of dependent resources triggered; waiting until dependent resources are deleted") if !reflect.DeepEqual(status.Inventory, savedStatus.Inventory) { r.backoff.Forget(req) } + status.SetState(StateDeleting, readyConditionReasonDeletionProcessing, "Deletion of dependent resources triggered; waiting until dependent resources are deleted") return ctrl.Result{RequeueAfter: r.backoff.Next(req, readyConditionReasonDeletionProcessing)}, nil } } diff --git a/pkg/component/reference.go b/pkg/component/reference.go index 3330d5e..a1b4079 100644 --- a/pkg/component/reference.go +++ b/pkg/component/reference.go @@ -7,7 +7,6 @@ package component import ( "context" - "encoding/json" "fmt" "reflect" "strings" @@ -69,8 +68,7 @@ func (r *ConfigMapReference) digest() string { if !r.loaded { return "" } - // note: this must() is ok because marshalling map[string]string should always work - return sha256hex(must(json.Marshal(r.data))) + return calculateDigest(r.data) } // Return the previously loaded configmap data. @@ -176,8 +174,7 @@ func (r *SecretReference) digest() string { if !r.loaded { return "" } - // note: this must() is ok because marshalling map[string][]byte should always work - return sha256hex(must(json.Marshal(r.data))) + return calculateDigest(r.data) } // Return the previously loaded secret data. diff --git a/pkg/component/target.go b/pkg/component/target.go index a3d0edf..af8c2ca 100644 --- a/pkg/component/target.go +++ b/pkg/component/target.go @@ -33,7 +33,7 @@ func newReconcileTarget[T Component](reconcilerName string, reconcilerId string, } } -func (t *reconcileTarget[T]) Apply(ctx context.Context, component T) (bool, error) { +func (t *reconcileTarget[T]) Apply(ctx context.Context, component T) (bool, string, error) { //log := log.FromContext(ctx) namespace := "" name := "" @@ -59,10 +59,12 @@ func (t *reconcileTarget[T]) Apply(ctx context.Context, component T) (bool, erro WithComponentDigest(componentDigest) objects, err := t.resourceGenerator.Generate(generateCtx, namespace, name, component.GetSpec()) if err != nil { - return false, errors.Wrap(err, "error rendering manifests") + return false, "", errors.Wrap(err, "error rendering manifests") } - return t.reconciler.Apply(ctx, &status.Inventory, objects, namespace, ownerId, component.GetGeneration()) + ok, err := t.reconciler.Apply(ctx, &status.Inventory, objects, namespace, ownerId, component.GetGeneration()) + + return ok, calculateDigest(componentDigest, objects), err } func (t *reconcileTarget[T]) Delete(ctx context.Context, component T) (bool, error) { diff --git a/pkg/component/types.go b/pkg/component/types.go index 1bf35fe..43f72a0 100644 --- a/pkg/component/types.go +++ b/pkg/component/types.go @@ -77,6 +77,13 @@ type RetryConfiguration interface { GetRetryInterval() time.Duration } +// The TimeoutConfiguration interface is meant to be implemented by components (or their spec) which offer +// tweaking the processing timeout (by default, it would be the value of the requeue interval). +type TimeoutConfiguration interface { + // Get timeout. Should be greater than 1 minute. + GetTimeout() time.Duration +} + // +kubebuilder:object:generate=true // Legacy placement spec. Components may include this into their spec. @@ -147,12 +154,27 @@ var _ RetryConfiguration = &RetrySpec{} // +kubebuilder:object:generate=true +// TimeoutSpec defines the processing timeout, that is, the duration after which all dependent objects of the component +// must have reached a ready state, or the component status will change to error. +// Components providing TimeoutConfiguration may include this into their spec. +type TimeoutSpec struct { + // +kubebuilder:validation:Type:=string + // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$" + Timeout *metav1.Duration `json:"timeout,omitempty"` +} + +var _ TimeoutConfiguration = &TimeoutSpec{} + +// +kubebuilder:object:generate=true + // Component Status. Components must include this into their status. type Status struct { ObservedGeneration int64 `json:"observedGeneration"` AppliedGeneration int64 `json:"appliedGeneration,omitempty"` LastObservedAt *metav1.Time `json:"lastObservedAt,omitempty"` LastAppliedAt *metav1.Time `json:"lastAppliedAt,omitempty"` + ProcessingDigest string `json:"processingDigest,omitempty"` + ProcessingSince *metav1.Time `json:"processingSince,omitempty"` Conditions []Condition `json:"conditions,omitempty"` // +kubebuilder:validation:Enum=Ready;Pending;Processing;DeletionPending;Deleting;Error State State `json:"state,omitempty"` diff --git a/pkg/component/util.go b/pkg/component/util.go index 8cbad17..0856e1e 100644 --- a/pkg/component/util.go +++ b/pkg/component/util.go @@ -8,6 +8,7 @@ package component import ( "crypto/sha256" "encoding/hex" + "encoding/json" "math/rand" "strings" "time" @@ -31,6 +32,11 @@ func sha256hex(data []byte) string { return hex.EncodeToString(sum[:]) } +func calculateDigest(values ...any) string { + // note: this must() is ok because the input values are expected to be JSON values + return sha256hex(must(json.Marshal(values))) +} + func capitalize(s string) string { if len(s) <= 1 { return s diff --git a/pkg/component/zz_generated.deepcopy.go b/pkg/component/zz_generated.deepcopy.go index 611af48..72d2dea 100644 --- a/pkg/component/zz_generated.deepcopy.go +++ b/pkg/component/zz_generated.deepcopy.go @@ -439,6 +439,10 @@ func (in *Status) DeepCopyInto(out *Status) { in, out := &in.LastAppliedAt, &out.LastAppliedAt *out = (*in).DeepCopy() } + if in.ProcessingSince != nil { + in, out := &in.ProcessingSince, &out.ProcessingSince + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]Condition, len(*in)) @@ -468,3 +472,23 @@ func (in *Status) DeepCopy() *Status { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeoutSpec) DeepCopyInto(out *TimeoutSpec) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeoutSpec. +func (in *TimeoutSpec) DeepCopy() *TimeoutSpec { + if in == nil { + return nil + } + out := new(TimeoutSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/reconciler/util.go b/pkg/reconciler/util.go index aa0f46d..79bbd1c 100644 --- a/pkg/reconciler/util.go +++ b/pkg/reconciler/util.go @@ -95,6 +95,7 @@ func calculateObjectDigest(obj client.Object, item *InventoryItem, revision int6 } now := time.Now().Unix() timestamp := int64(0) + // TODO: make force-reconcile period configurable (globally, per object, ...?) if previousDigest == digest && now-previousTimestamp <= 3600 { timestamp = previousTimestamp } else { diff --git a/pkg/status/analyzer_test.go b/pkg/status/analyzer_test.go index eacec64..fdda642 100644 --- a/pkg/status/analyzer_test.go +++ b/pkg/status/analyzer_test.go @@ -26,115 +26,110 @@ var _ = Describe("testing: analyzer.go", func() { analyzer = status.NewStatusAnalyzer("test") }) - Context("xxx", func() { - BeforeEach(func() { - }) - - DescribeTable("testing: ComputeStatus()", - func(generation int, observedGeneration int, conditions map[kstatus.ConditionType]corev1.ConditionStatus, hintObservedGeneration bool, hintReadyCondition bool, hintConditions []string, expectedStatus status.Status) { - obj := Object{ - ObjectMeta: metav1.ObjectMeta{ - Generation: int64(generation), - }, - Status: ObjectStatus{ - ObservedGeneration: int64(observedGeneration), - }, + DescribeTable("testing: ComputeStatus()", + func(generation int, observedGeneration int, conditions map[kstatus.ConditionType]corev1.ConditionStatus, hintObservedGeneration bool, hintReadyCondition bool, hintConditions []string, expectedStatus status.Status) { + obj := Object{ + ObjectMeta: metav1.ObjectMeta{ + Generation: int64(generation), + }, + Status: ObjectStatus{ + ObservedGeneration: int64(observedGeneration), + }, + } + for name, status := range conditions { + obj.Status.Conditions = append(obj.Status.Conditions, kstatus.Condition{ + Type: name, + Status: status, + }) + } + var hints []string + if hintObservedGeneration { + hints = append(hints, "has-observed-generation") + } + if hintReadyCondition { + hints = append(hints, "has-ready-condition") + } + if len(hintConditions) > 0 { + hints = append(hints, "conditions="+strings.Join(hintConditions, ";")) + } + if len(hints) > 0 { + obj.Annotations = map[string]string{ + "test/status-hint": strings.Join(hints, ","), } - for name, status := range conditions { - obj.Status.Conditions = append(obj.Status.Conditions, kstatus.Condition{ - Type: name, - Status: status, - }) - } - var hints []string - if hintObservedGeneration { - hints = append(hints, "has-observed-generation") - } - if hintReadyCondition { - hints = append(hints, "has-ready-condition") - } - if len(hintConditions) > 0 { - hints = append(hints, "conditions="+strings.Join(hintConditions, ";")) - } - if len(hints) > 0 { - obj.Annotations = map[string]string{ - "test/status-hint": strings.Join(hints, ","), - } - } - unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) - Expect(err).NotTo(HaveOccurred()) - unstructuredObj := &unstructured.Unstructured{Object: unstructuredContent} - - computedStatus, err := analyzer.ComputeStatus(unstructuredObj) - Expect(err).NotTo(HaveOccurred()) - - Expect(computedStatus).To(Equal(expectedStatus)) - }, - - Entry(nil, 3, 0, nil, false, false, nil, status.CurrentStatus), - Entry(nil, 3, 1, nil, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, nil, false, false, nil, status.CurrentStatus), - Entry(nil, 3, 0, nil, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, nil, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, nil, true, false, nil, status.CurrentStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.CurrentStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.CurrentStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.CurrentStatus), - - Entry(nil, 3, 0, nil, false, true, nil, status.InProgressStatus), - Entry(nil, 3, 1, nil, false, true, nil, status.InProgressStatus), - Entry(nil, 3, 3, nil, false, true, nil, status.InProgressStatus), - Entry(nil, 3, 0, nil, true, true, nil, status.InProgressStatus), - Entry(nil, 3, 1, nil, true, true, nil, status.InProgressStatus), - Entry(nil, 3, 3, nil, true, true, nil, status.InProgressStatus), - - Entry(nil, 3, 0, nil, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, nil, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, nil, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 0, nil, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, nil, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, nil, true, false, []string{"Test"}, status.InProgressStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), - - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.CurrentStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.CurrentStatus), - Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.InProgressStatus), - Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.CurrentStatus), - ) - }) + } + unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) + Expect(err).NotTo(HaveOccurred()) + unstructuredObj := &unstructured.Unstructured{Object: unstructuredContent} + + computedStatus, err := analyzer.ComputeStatus(unstructuredObj) + Expect(err).NotTo(HaveOccurred()) + + Expect(computedStatus).To(Equal(expectedStatus)) + }, + + Entry(nil, 3, 0, nil, false, false, nil, status.CurrentStatus), + Entry(nil, 3, 1, nil, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, nil, false, false, nil, status.CurrentStatus), + Entry(nil, 3, 0, nil, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, nil, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, nil, true, false, nil, status.CurrentStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionUnknown}, true, false, nil, status.InProgressStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionFalse}, true, false, nil, status.InProgressStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.CurrentStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, false, false, nil, status.CurrentStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Ready": corev1.ConditionTrue}, true, false, nil, status.CurrentStatus), + + Entry(nil, 3, 0, nil, false, true, nil, status.InProgressStatus), + Entry(nil, 3, 1, nil, false, true, nil, status.InProgressStatus), + Entry(nil, 3, 3, nil, false, true, nil, status.InProgressStatus), + Entry(nil, 3, 0, nil, true, true, nil, status.InProgressStatus), + Entry(nil, 3, 1, nil, true, true, nil, status.InProgressStatus), + Entry(nil, 3, 3, nil, true, true, nil, status.InProgressStatus), + + Entry(nil, 3, 0, nil, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, nil, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, nil, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 0, nil, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, nil, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, nil, true, false, []string{"Test"}, status.InProgressStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionFalse}, true, false, []string{"Test"}, status.InProgressStatus), + + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.CurrentStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, false, false, []string{"Test"}, status.CurrentStatus), + Entry(nil, 3, 0, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 1, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.InProgressStatus), + Entry(nil, 3, 3, map[kstatus.ConditionType]corev1.ConditionStatus{"Test": corev1.ConditionTrue}, true, false, []string{"Test"}, status.CurrentStatus), + ) }) type Object struct { diff --git a/website/content/en/docs/concepts/reconciler.md b/website/content/en/docs/concepts/reconciler.md index 7d7e196..cb63787 100644 --- a/website/content/en/docs/concepts/reconciler.md +++ b/website/content/en/docs/concepts/reconciler.md @@ -170,4 +170,24 @@ type RequeueConfiguration interface { } ``` +interface. + +## Tuning the timeout behavior + +If the dependent objects of a component do not reach a ready state after a certain time, the component state will switch from `Processing` to `Error`. +This timeout restarts counting whenever something changed in the component, or in the manifests of the dependent objects, and by default has the value +of the effective retry interval, which in turn defaults to 10 minutes. +The timeout may be overridden by the component by implementing the + +```go +package component + +// The TimeoutConfiguration interface is meant to be implemented by components (or their spec) which offer +// tweaking the processing timeout (by default, it would be the value of the requeue interval). +type TimeoutConfiguration interface { + // Get timeout. Should be greater than 1 minute. + GetTimeout() time.Duration +} +``` + interface. \ No newline at end of file