fix: enabling deleted grafana datasources to be recreated

Prior to this fix, grafana datasources which were deleted were not
being recreated.

pkg/controllers/monitoring-stack/components.go

@@ -318,15 +318,21 @@ func stackComponentPatchers(ms *stack.MonitoringStack, instanceSelectorKey strin
 func newGrafanaDataSource(ms *stack.MonitoringStack) *grafanav1alpha1.GrafanaDataSource {
 	datasourceName := fmt.Sprintf("ms-%s-%s", ms.Namespace, ms.Name)
 	prometheusURL := fmt.Sprintf("%s-prometheus.%s:9090", ms.Name, ms.Namespace)
+	annotations := map[string]string{
+		grafanaDatasourceOwnerName:      ms.Name,
+		grafanaDatasourceOwnerNamespace: ms.Namespace,
+	}
+
 	return &grafanav1alpha1.GrafanaDataSource{
 		TypeMeta: metav1.TypeMeta{
 			// NOTE: uses a different naming convention for SchemeGroupVersion
 			APIVersion: grafanav1alpha1.GroupVersion.String(),
 			Kind:       "GrafanaDataSource",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      datasourceName,
-			Namespace: grafana_operator.Namespace,
+			Name:        datasourceName,
+			Namespace:   grafana_operator.Namespace,
+			Annotations: annotations,
 		},
 		Spec: grafanav1alpha1.GrafanaDataSourceSpec{
 			Name: datasourceName,

pkg/controllers/monitoring-stack/controller.go

@@ -22,6 +22,12 @@ import (
 	"strings"
 	"time"
 
+	grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
 	policyv1 "k8s.io/api/policy/v1"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -42,12 +48,19 @@ import (
 	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 )
 
+const (
+	grafanaDatasourceOwnerName      = "monitoring-stack-operator/owner-name"
+	grafanaDatasourceOwnerNamespace = "monitoring-stack-operator/owner-namespace"
+)
+
 type reconciler struct {
-	k8sClient             client.Client
-	scheme                *runtime.Scheme
-	logger                logr.Logger
-	instanceSelectorKey   string
-	instanceSelectorValue string
+	k8sClient                  client.Client
+	scheme                     *runtime.Scheme
+	logger                     logr.Logger
+	instanceSelectorKey        string
+	instanceSelectorValue      string
+	shouldCreateGrafanaDSWatch bool
+	controller                 controller.Controller
 }
 
 // Options allows for controller options to be set
@@ -80,28 +93,35 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error {
 	}
 
 	r := &reconciler{
-		k8sClient:             mgr.GetClient(),
-		scheme:                mgr.GetScheme(),
-		logger:                ctrl.Log.WithName("monitoring-stack-operator"),
-		instanceSelectorKey:   split[0],
-		instanceSelectorValue: split[1],
+		k8sClient:                  mgr.GetClient(),
+		scheme:                     mgr.GetScheme(),
+		logger:                     ctrl.Log.WithName("monitoring-stack-operator"),
+		instanceSelectorKey:        split[0],
+		instanceSelectorValue:      split[1],
+		shouldCreateGrafanaDSWatch: true,
 	}
 
 	// We only want to trigger a reconciliation when the generation
 	// of a child changes. Until we need to update our the status for our own objects,
 	// we can save CPU cycles by avoiding reconciliations triggered by
 	// child status changes.
 	p := predicate.GenerationChangedPredicate{}
-	return ctrl.NewControllerManagedBy(mgr).
+
+	ctrl, err := ctrl.NewControllerManagedBy(mgr).
 		WithLogger(ctrl.Log).
 		For(&stack.MonitoringStack{}).
 		Owns(&monv1.Prometheus{}).WithEventFilter(p).
 		Owns(&v1.ServiceAccount{}).WithEventFilter(p).
 		Owns(&rbacv1.Role{}).WithEventFilter(p).
 		Owns(&rbacv1.RoleBinding{}).WithEventFilter(p).
 		Owns(&monv1.ServiceMonitor{}).WithEventFilter(p).
-		Owns(&policyv1.PodDisruptionBudget{}).WithEventFilter(p).
-		Complete(r)
+		Owns(&policyv1.PodDisruptionBudget{}).WithEventFilter(p).Build(r)
+
+	if err != nil {
+		return err
+	}
+	r.controller = ctrl
+	return nil
 }
 
 func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -122,6 +142,10 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, err
 	}
 
+	if err = r.createGrafanaDSWatch(ctx); err != nil {
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
 	for _, patcher := range patchers {
 
 		err := r.reconcileObject(ctx, ms, patcher)
@@ -136,7 +160,6 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 			return ctrl.Result{}, err
 
 		}
-
 	}
 
 	return ctrl.Result{}, nil
@@ -196,3 +219,34 @@ func (r *reconciler) reconcileObject(ctx context.Context, ms *stack.MonitoringSt
 	logger.Info("Updating stack component")
 	return r.k8sClient.Update(ctx, desired)
 }
+
+func (r *reconciler) createGrafanaDSWatch(ctx context.Context) error {
+	if r.shouldCreateGrafanaDSWatch {
+		log := r.logger.WithName("create-grafana-ds-watch")
+		var (
+			dataSources grafanav1alpha1.GrafanaDataSourceList
+		)
+		err := r.k8sClient.List(ctx, &dataSources, client.InNamespace("default"))
+
+		if err != nil {
+			log.V(3).Info("grafana data source CRD is not defined")
+			return err
+		}
+
+		err = r.controller.Watch(&source.Kind{Type: &grafanav1alpha1.GrafanaDataSource{}}, handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {
+			name := object.GetAnnotations()[grafanaDatasourceOwnerName]
+			namespace := object.GetAnnotations()[grafanaDatasourceOwnerNamespace]
+			namespaceName := types.NamespacedName{Name: name, Namespace: namespace}
+			return []reconcile.Request{{NamespacedName: namespaceName}}
+		}))
+		if err != nil {
+			log.V(3).Info("unable to create watch on grafana data source")
+			return err
+		}
+
+		log.V(6).Info("Created watch on Grafana datasource")
+		r.shouldCreateGrafanaDSWatch = false
+	}
+
+	return nil
+}

test/e2e/monitoring_stack_controller_test.go

@@ -23,6 +23,7 @@ import (
 
 	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 
+	grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1"
 	"gotest.tools/v3/assert"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -78,6 +79,9 @@ func TestMonitoringStackController(t *testing.T) {
 				assertAlertmanagersAreOnDifferentNodes(t, pods)
 				assertAlertmanagersAreResilientToDisruption(t, pods)
 			},
+		}, {
+			name:     "Grafana data source is recreated on delete",
+			scenario: recreateDeleteGrafanaDS,
 		},
 	}
 
@@ -163,6 +167,24 @@ func reconcileRevertsManualChanges(t *testing.T) {
 	assert.DeepEqual(t, generated.Spec, reconciled.Spec)
 }
 
+func recreateDeleteGrafanaDS(t *testing.T) {
+	ms := newMonitoringStack(t, "test-grafana-ds")
+	datasourceName := fmt.Sprintf("ms-%s-%s", ms.Namespace, ms.Name)
+
+	err := f.K8sClient.Create(context.Background(), ms)
+	assert.NilError(t, err, "failed to create a monitoring stack")
+
+	grafanaDS := grafanav1alpha1.GrafanaDataSource{}
+
+	f.GetResourceWithRetry(t, datasourceName, "monitoring-stack-operator", &grafanaDS)
+
+	assert.NilError(t, err, "grafana data source is not created")
+
+	f.K8sClient.Delete(context.Background(), &grafanaDS)
+
+	f.GetResourceWithRetry(t, datasourceName, "monitoring-stack-operator", &grafanaDS)
+}
+
 func validateStackLogLevel(t *testing.T) {
 	invalidLogLevels := []string{
 		"foobar",