feat(tfrun): Add cleanup of past TerraformRuns (#168)

* feat(tfrun): wip: create a cleanup routine in layer ctrl

* feat: implement tfrun cleanup based on number of runs to keep

---------

Co-authored-by: Alan <alanl@padok.fr>

api/v1alpha1/common.go

@@ -5,6 +5,11 @@ import (
 	resource "k8s.io/apimachinery/pkg/api/resource"
 )
 
+const (
+	PlanRunRetention  int = 6
+	ApplyRunRetention int = 6
+)
+
 type OverrideRunnerSpec struct {
 	ImagePullSecrets   []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 	Image              string                        `json:"image,omitempty"`
@@ -24,6 +29,11 @@ type MetadataOverride struct {
 	Labels      map[string]string `json:"labels,omitempty"`
 }
 
+type RunHistoryPolicy struct {
+	KeepLastPlanRuns  *int `json:"plan,omitempty"`
+	KeepLastApplyRuns *int `json:"apply,omitempty"`
+}
+
 type RemediationStrategy struct {
 	AutoApply bool                       `json:"autoApply,omitempty"`
 	OnError   OnErrorRemediationStrategy `json:"onError,omitempty"`
@@ -89,6 +99,13 @@ func GetOverrideRunnerSpec(repository *TerraformRepository, layer *TerraformLaye
 	}
 }
 
+func GetRunHistoryPolicy(repository *TerraformRepository, layer *TerraformLayer) RunHistoryPolicy {
+	return RunHistoryPolicy{
+		KeepLastPlanRuns:  chooseInt(repository.Spec.RunHistoryPolicy.KeepLastPlanRuns, layer.Spec.RunHistoryPolicy.KeepLastPlanRuns, PlanRunRetention),
+		KeepLastApplyRuns: chooseInt(repository.Spec.RunHistoryPolicy.KeepLastApplyRuns, layer.Spec.RunHistoryPolicy.KeepLastApplyRuns, ApplyRunRetention),
+	}
+}
+
 func mergeImagePullSecrets(a, b []corev1.LocalObjectReference) []corev1.LocalObjectReference {
 	result := []corev1.LocalObjectReference{}
 	temp := map[string]string{}
@@ -113,6 +130,16 @@ func chooseString(a, b string) string {
 	return a
 }
 
+func chooseInt(a, b *int, d int) *int {
+	if b != nil {
+		return b
+	}
+	if a != nil {
+		return a
+	}
+	return &d
+}
+
 func mergeEnvFrom(a, b []corev1.EnvFromSource) []corev1.EnvFromSource {
 	result := []corev1.EnvFromSource{}
 	tempSecret := map[string]string{}

api/v1alpha1/common_test.go

@@ -1371,3 +1371,84 @@ func TestOverrideRunnerSpec(t *testing.T) {
 		})
 	}
 }
+
+func intPointer(i int) *int {
+	return &i
+}
+
+func TestGetHistoryPolicy(t *testing.T) {
+	tt := []struct {
+		name                  string
+		repository            *configv1alpha1.TerraformRepository
+		layer                 *configv1alpha1.TerraformLayer
+		expectedHistoryPolicy configv1alpha1.RunHistoryPolicy
+	}{
+		{
+			"OnlyRepositoryHistoryPolicy",
+			&configv1alpha1.TerraformRepository{
+				Spec: configv1alpha1.TerraformRepositorySpec{
+					RunHistoryPolicy: configv1alpha1.RunHistoryPolicy{
+						KeepLastPlanRuns:  intPointer(10),
+						KeepLastApplyRuns: intPointer(10),
+					},
+				},
+			},
+			&configv1alpha1.TerraformLayer{},
+			configv1alpha1.RunHistoryPolicy{
+				KeepLastPlanRuns:  intPointer(10),
+				KeepLastApplyRuns: intPointer(10),
+			},
+		},
+		{
+			"OnlyLayerHistoryPolicy",
+			&configv1alpha1.TerraformRepository{},
+			&configv1alpha1.TerraformLayer{
+				Spec: configv1alpha1.TerraformLayerSpec{
+					RunHistoryPolicy: configv1alpha1.RunHistoryPolicy{
+						KeepLastPlanRuns:  intPointer(10),
+						KeepLastApplyRuns: intPointer(10),
+					},
+				},
+			},
+			configv1alpha1.RunHistoryPolicy{
+				KeepLastPlanRuns:  intPointer(10),
+				KeepLastApplyRuns: intPointer(10),
+			},
+		},
+		{
+			"OverrideRepositoryWithLayer",
+			&configv1alpha1.TerraformRepository{
+				Spec: configv1alpha1.TerraformRepositorySpec{
+					RunHistoryPolicy: configv1alpha1.RunHistoryPolicy{
+						KeepLastPlanRuns:  intPointer(10),
+						KeepLastApplyRuns: intPointer(10),
+					},
+				},
+			},
+			&configv1alpha1.TerraformLayer{
+				Spec: configv1alpha1.TerraformLayerSpec{
+					RunHistoryPolicy: configv1alpha1.RunHistoryPolicy{
+						KeepLastPlanRuns:  intPointer(6),
+						KeepLastApplyRuns: intPointer(5),
+					},
+				},
+			},
+			configv1alpha1.RunHistoryPolicy{
+				KeepLastPlanRuns:  intPointer(6),
+				KeepLastApplyRuns: intPointer(5),
+			},
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			result := configv1alpha1.GetRunHistoryPolicy(tc.repository, tc.layer)
+			if *tc.expectedHistoryPolicy.KeepLastPlanRuns != *result.KeepLastPlanRuns {
+				t.Errorf("different plan policy computed: expected %d got %d", *tc.expectedHistoryPolicy.KeepLastPlanRuns, *result.KeepLastPlanRuns)
+			}
+			if *tc.expectedHistoryPolicy.KeepLastApplyRuns != *result.KeepLastApplyRuns {
+				t.Errorf("different apply policy computed: expected %d got %d", *tc.expectedHistoryPolicy.KeepLastApplyRuns, *result.KeepLastApplyRuns)
+			}
+		})
+	}
+}

api/v1alpha1/terraformlayer_types.go

@@ -34,6 +34,7 @@ type TerraformLayerSpec struct {
 	Repository          TerraformLayerRepository `json:"repository,omitempty"`
 	RemediationStrategy RemediationStrategy      `json:"remediationStrategy,omitempty"`
 	OverrideRunnerSpec  OverrideRunnerSpec       `json:"overrideRunnerSpec,omitempty"`
+	RunHistoryPolicy    RunHistoryPolicy         `json:"runHistoryPolicy,omitempty"`
 }
 
 type TerraformLayerRepository struct {

api/v1alpha1/terraformrepository_types.go

@@ -32,6 +32,7 @@ type TerraformRepositorySpec struct {
 	TerraformConfig     TerraformConfig               `json:"terraform,omitempty"`
 	RemediationStrategy RemediationStrategy           `json:"remediationStrategy,omitempty"`
 	OverrideRunnerSpec  OverrideRunnerSpec            `json:"overrideRunnerSpec,omitempty"`
+	RunHistoryPolicy    RunHistoryPolicy              `json:"runHistoryPolicy,omitempty"`
 }
 
 type TerraformRepositoryRepository struct {

internal/controllers/terraformlayer/controller.go

@@ -104,6 +104,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		log.Errorf("failed to get TerraformRepository linked to layer %s: %s", layer.Name, err)
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}, err
 	}
+	err = r.cleanupRuns(ctx, layer, repository)
+	if err != nil {
+		log.Warningf("failed to cleanup runs for layer %s: %s", layer.Name, err)
+	}
 	state, conditions := r.GetState(ctx, layer)
 	lastResult, err := r.Storage.Get(storage.GenerateKey(storage.LastPlanResult, layer))
 	if err != nil {

internal/controllers/terraformlayer/controller_test.go

@@ -368,6 +368,10 @@ var _ = Describe("Layer", func() {
 			})
 		})
 	})
+	// TODO: test cleanup of runs
+	Describe("Cleanup case", func() {
+
+	})
 })
 
 var _ = AfterSuite(func() {

internal/controllers/terraformlayer/run.go

@@ -1,10 +1,18 @@
 package terraformlayer
 
 import (
+	"context"
 	"fmt"
+	"sort"
+	"sync"
+	"time"
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
+	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type Action string
@@ -44,3 +52,119 @@ func (r *Reconciler) getRun(layer *configv1alpha1.TerraformLayer, repository *co
 		},
 	}
 }
+
+func (r *Reconciler) getAllFinishedRuns(ctx context.Context, layer *configv1alpha1.TerraformLayer, repository *configv1alpha1.TerraformRepository) ([]*configv1alpha1.TerraformRun, error) {
+	list := &configv1alpha1.TerraformRunList{}
+	labelSelector := labels.NewSelector()
+	for key, value := range GetDefaultLabels(layer) {
+		requirement, err := labels.NewRequirement(key, selection.Equals, []string{value})
+		if err != nil {
+			return []*configv1alpha1.TerraformRun{}, err
+		}
+		labelSelector = labelSelector.Add(*requirement)
+	}
+	err := r.Client.List(
+		ctx,
+		list,
+		client.MatchingLabelsSelector{Selector: labelSelector},
+		&client.ListOptions{Namespace: layer.Namespace},
+	)
+	if err != nil {
+		return []*configv1alpha1.TerraformRun{}, err
+	}
+
+	// Keep only runs with state Succeeded or Failed
+	var runs []*configv1alpha1.TerraformRun
+	for _, run := range list.Items {
+		if run.Status.State == "Succeeded" || run.Status.State == "Failed" {
+			runs = append(runs, &run)
+		}
+	}
+	return runs, nil
+}
+
+type runRetention struct {
+	plan  time.Duration
+	apply time.Duration
+}
+
+func deleteAll(ctx context.Context, c client.Client, objs []*configv1alpha1.TerraformRun) error {
+	var wg sync.WaitGroup
+	errorCh := make(chan error, len(objs))
+
+	deleteObject := func(obj *configv1alpha1.TerraformRun) {
+		defer wg.Done()
+		err := c.Delete(ctx, obj)
+		if err != nil {
+			errorCh <- fmt.Errorf("error deleting %s: %v", obj.Name, err)
+		} else {
+			log.Infof("deleted run %s", obj.Name)
+		}
+	}
+
+	for _, obj := range objs {
+		wg.Add(1)
+		go deleteObject(obj)
+	}
+
+	go func() {
+		wg.Wait()
+		close(errorCh)
+	}()
+
+	var ret error = nil
+	for err := range errorCh {
+		if err != nil {
+			ret = err
+		}
+	}
+
+	return ret
+}
+
+func (r *Reconciler) cleanupRuns(ctx context.Context, layer *configv1alpha1.TerraformLayer, repository *configv1alpha1.TerraformRepository) error {
+	historyPolicy := configv1alpha1.GetRunHistoryPolicy(repository, layer)
+
+	runs, err := r.getAllFinishedRuns(ctx, layer, repository)
+	if err != nil {
+		return err
+	}
+	sortedRuns := sortAndSplitRunsByAction(runs)
+	toDelete := []*configv1alpha1.TerraformRun{}
+	if len(sortedRuns[string(PlanAction)]) <= *historyPolicy.KeepLastPlanRuns {
+		log.Infof("no plan runs to delete for layer %s", layer.Name)
+	} else {
+		toDelete = append(toDelete, sortedRuns[string(PlanAction)][:len(sortedRuns[string(PlanAction)])-*historyPolicy.KeepLastPlanRuns]...)
+	}
+	if len(sortedRuns[string(ApplyAction)]) <= *historyPolicy.KeepLastApplyRuns {
+		log.Infof("no apply runs to delete for layer %s", layer.Name)
+	} else {
+		toDelete = append(toDelete, sortedRuns[string(ApplyAction)][:len(sortedRuns[string(ApplyAction)])-*historyPolicy.KeepLastApplyRuns]...)
+	}
+	if len(toDelete) == 0 {
+		log.Infof("no runs to delete for layer %s", layer.Name)
+		return nil
+	}
+	err = deleteAll(ctx, r.Client, toDelete)
+	if err != nil {
+		return err
+	}
+	log.Infof("deleted %d runs for layer %s", len(toDelete), layer.Name)
+	return nil
+}
+
+func sortAndSplitRunsByAction(runs []*configv1alpha1.TerraformRun) map[string][]*configv1alpha1.TerraformRun {
+	splittedRuns := map[string][]*configv1alpha1.TerraformRun{}
+	for _, run := range runs {
+		if _, ok := splittedRuns[run.Spec.Action]; !ok {
+			splittedRuns[run.Spec.Action] = []*configv1alpha1.TerraformRun{}
+		}
+		splittedRuns[run.Spec.Action] = append(splittedRuns[run.Spec.Action], run)
+	}
+	for action, _ := range splittedRuns {
+		sort.Slice(splittedRuns[action], func(i, j int) bool {
+			return splittedRuns[action][i].CreationTimestamp.Before(&splittedRuns[action][j].CreationTimestamp)
+		})
+	}
+	return splittedRuns
+}

internal/controllers/terraformlayer/testdata/cleanup-case.yaml

@@ -0,0 +1,22 @@
+apiVersion: config.terraform.padok.cloud/v1alpha1
+kind: TerraformRepository
+metadata:
+  name: cleanup-repo-1
+  namespace: default
+spec:
+  repository:
+    url: git@github.com:padok-team/burrito-examples.git
+---
+apiVersion: config.terraform.padok.cloud/v1alpha1
+kind: TerraformLayer
+metadata:
+  name: cleanup-case-1
+  namespace: default
+spec:
+  branch: main
+  path: cleanup-case-1/
+  repository:
+    name: cleanup-repo-1
+    namespace: default
+  terraform:
+    version: 1.3.1