Merge pull request #250 from fluxcd/retry-interval

Allow failed reconciliations to be scheduled at a different interval
fluxcd · Jan 22, 2021 · 89017ca · 89017ca
2 parents ba979af + 0e0277a
commit 89017ca
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 8 deletions.
diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go
@@ -48,6 +48,12 @@ type KustomizationSpec struct {
 	// +required
 	Interval metav1.Duration `json:"interval"`
 
+	// The interval at which to retry a previously failed reconciliation.
+	// When not specified, the controller uses the KustomizationSpec.Interval
+	// value to retry failures.
+	// +optional
+	RetryInterval *metav1.Duration `json:"retryInterval,omitempty"`
+
 	// The KubeConfig for reconciling the Kustomization on a remote cluster.
 	// When specified, KubeConfig takes precedence over ServiceAccountName.
 	// +optional
@@ -224,6 +230,14 @@ func (in Kustomization) GetTimeout() time.Duration {
 	return duration
 }
 
+// GetRetryInterval returns the retry interval
+func (in Kustomization) GetRetryInterval() time.Duration {
+	if in.Spec.RetryInterval != nil {
+		return in.Spec.RetryInterval.Duration
+	}
+	return in.Spec.Interval.Duration
+}
+
 func (in Kustomization) GetDependsOn() (types.NamespacedName, []dependency.CrossNamespaceDependencyReference) {
 	return types.NamespacedName{
 		Namespace: in.Namespace,

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
@@ -171,6 +171,11 @@ spec:
               prune:
                 description: Prune enables garbage collection.
                 type: boolean
+              retryInterval:
+                description: The interval at which to retry a previously failed reconciliation.
+                  When not specified, the controller uses the KustomizationSpec.Interval
+                  value to retry failures.
+                type: string
               serviceAccountName:
                 description: The name of the Kubernetes service account to impersonate
                   when reconciling this Kustomization.

diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go
@@ -226,20 +226,20 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	(logr.FromContext(ctx)).Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
-		time.Now().Sub(reconcileStart).String(),
-		kustomization.Spec.Interval.Duration.String()),
-		"revision",
-		source.GetArtifact().Revision,
-	)
-
 	// requeue
 	if reconcileErr != nil {
 		// record the reconciliation error
 		r.recordReadiness(ctx, reconciledKustomization)
-		return ctrl.Result{RequeueAfter: kustomization.Spec.Interval.Duration}, reconcileErr
+		return ctrl.Result{RequeueAfter: kustomization.GetRetryInterval()}, reconcileErr
 	}
 
+	log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s",
+		time.Now().Sub(reconcileStart).String(),
+		kustomization.Spec.Interval.Duration.String()),
+		"revision",
+		source.GetArtifact().Revision,
+	)
+
 	// record the reconciliation result
 	r.event(ctx, reconciledKustomization, source.GetArtifact().Revision, events.EventSeverityInfo,
 		"Update completed", map[string]string{"commit_status": "update"})

diff --git a/controllers/kustomization_controller_test.go b/controllers/kustomization_controller_test.go
@@ -190,6 +190,11 @@ var _ = Describe("KustomizationReconciler", func() {
 
 			Expect(cond.Status).To(Equal(t.expectStatus))
 			Expect(got.Status.LastAppliedRevision).To(Equal(t.expectRevision))
+
+			ns := &corev1.Namespace{}
+			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test"}, ns)).Should(Succeed())
+			Expect(ns.Labels[fmt.Sprintf("%s/name", kustomizev1.GroupVersion.Group)]).To(Equal(kName.Name))
+			Expect(ns.Labels[fmt.Sprintf("%s/namespace", kustomizev1.GroupVersion.Group)]).To(Equal(kName.Namespace))
 		},
 			Entry("namespace-sa", refTestCase{
 				artifacts: []testserver.File{

diff --git a/docs/api/kustomize.md b/docs/api/kustomize.md
@@ -113,6 +113,22 @@ Kubernetes meta/v1.Duration
 </tr>
 <tr>
 <td>
+<code>retryInterval</code><br>
+<em>
+<a href="https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">
+Kubernetes meta/v1.Duration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The interval at which to retry a previously failed reconciliation.
+When not specified, the controller uses the KustomizationSpec.Interval
+value to retry failures.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>kubeConfig</code><br>
 <em>
 <a href="#kustomize.toolkit.fluxcd.io/v1beta1.KubeConfig">
@@ -549,6 +565,22 @@ Kubernetes meta/v1.Duration
 </tr>
 <tr>
 <td>
+<code>retryInterval</code><br>
+<em>
+<a href="https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">
+Kubernetes meta/v1.Duration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The interval at which to retry a previously failed reconciliation.
+When not specified, the controller uses the KustomizationSpec.Interval
+value to retry failures.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>kubeConfig</code><br>
 <em>
 <a href="#kustomize.toolkit.fluxcd.io/v1beta1.KubeConfig">

diff --git a/docs/spec/v1beta1/kustomization.md b/docs/spec/v1beta1/kustomization.md
@@ -25,6 +25,12 @@ type KustomizationSpec struct {
 	// +required
 	Interval metav1.Duration `json:"interval"`
 
+	// The interval at which to retry a previously failed reconciliation.
+	// When not specified, the controller uses the KustomizationSpec.Interval
+	// value to retry failures.
+	// +optional
+	RetryInterval *metav1.Duration `json:"retryInterval,omitempty"`
+
 	// The KubeConfig for reconciling the Kustomization on a remote cluster.
 	// When specified, KubeConfig takes precedence over ServiceAccountName.
 	// +optional