handle update app object by removing an existing program

Signed-off-by: Mohamed Mahmoud <mmahmoud@redhat.com>

controllers/bpfman-agent/application-program.go

@@ -74,6 +74,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	for i, a := range appPrograms.Items {
+		var appProgramMap = make(map[string]bool)
 		for j, p := range a.Spec.Programs {
 			switch p.Type {
 			case bpfmaniov1alpha1.ProgTypeFentry:
@@ -93,6 +94,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				fentryObjects := []client.Object{&fentryProgram}
+				appProgramMap[fentryProgram.Name] = true
 				// Reconcile FentryProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, fentryObjects)
 
@@ -113,6 +115,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				fexitObjects := []client.Object{&fexitProgram}
+				appProgramMap[fexitProgram.Name] = true
 				// Reconcile FexitProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, fexitObjects)
 
@@ -134,6 +137,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				kprobeObjects := []client.Object{&kprobeProgram}
+				appProgramMap[kprobeProgram.Name] = true
 				// Reconcile KprobeProgram or KpretprobeProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, kprobeObjects)
 
@@ -155,6 +159,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				uprobeObjects := []client.Object{&uprobeProgram}
+				appProgramMap[uprobeProgram.Name] = true
 				// Reconcile UprobeProgram or UpretprobeProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, uprobeObjects)
 
@@ -175,6 +180,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				tracepointObjects := []client.Object{&tracepointProgram}
+				appProgramMap[tracepointProgram.Name] = true
 				// Reconcile TracepointProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, tracepointObjects)
 
@@ -202,6 +208,7 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				tcObjects := []client.Object{&tcProgram}
+				appProgramMap[tcProgram.Name] = true
 				// Reconcile TcProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, tcObjects)
 
@@ -228,16 +235,17 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				}
 				rec.appOwner = &a
 				xdpObjects := []client.Object{&xdpProgram}
+				appProgramMap[xdpProgram.Name] = true
 				// Reconcile XdpProgram.
 				complete, res, err = r.reconcileCommon(ctx, rec, xdpObjects)
 
 			default:
-				r.Logger.Error(fmt.Errorf("unsupported bpf program type"), "unsupported bpf program type", "ProgType", p.Type)
+				ctxLogger.Error(fmt.Errorf("unsupported bpf program type"), "unsupported bpf program type", "ProgType", p.Type)
 				// Skip this program and continue to the next one
 				continue
 			}
 
-			r.Logger.V(1).Info("Reconcile Application", "Application", i, "Program", j, "Name", a.Name,
+			ctxLogger.V(1).Info("Reconcile Application", "Application", i, "Program", j, "Name", a.Name,
 				"type", p.Type, "Complete", complete, "Result", res, "Error", err)
 
 			if complete {
@@ -249,6 +257,27 @@ func (r *BpfApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		}
 
 		if complete {
+			bpfPrograms := &bpfmaniov1alpha1.BpfProgramList{}
+			bpfDeletedPrograms := &bpfmaniov1alpha1.BpfProgramList{}
+			// find programs that need to be deleted and delete them
+			opts := []client.ListOption{client.MatchingLabels{internal.BpfProgramOwnerLabel: a.Name}}
+			if err := r.List(ctx, bpfPrograms, opts...); err != nil {
+				ctxLogger.Error(err, "failed to get freshPrograms for full reconcile")
+				return ctrl.Result{}, err
+			}
+			for _, bpfProgram := range bpfPrograms.Items {
+				progName := bpfProgram.Labels[internal.BpfParentProgram]
+				if _, ok := appProgramMap[progName]; !ok {
+					ctxLogger.Info("Deleting BpfProgram", "BpfProgram", progName)
+					bpfDeletedPrograms.Items = append(bpfDeletedPrograms.Items, bpfProgram)
+				}
+			}
+			// Delete BpfPrograms that are no longer needed
+			res, err = r.unLoadAndDeleteBpfProgramsList(ctx, bpfDeletedPrograms, internal.BpfApplicationControllerFinalizer)
+			if err != nil {
+				ctxLogger.Error(err, "failed to delete programs")
+				return ctrl.Result{}, err
+			}
 			// We've completed reconciling all programs for this application, continue to the next one
 			continue
 		} else {

controllers/bpfman-agent/common.go

@@ -36,6 +36,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1"
 	bpfmanagentinternal "github.com/bpfman/bpfman-operator/controllers/bpfman-agent/internal"
@@ -61,7 +62,8 @@ import (
 //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get
 
 const (
-	retryDurationAgent = 5 * time.Second
+	retryDurationAgent     = 5 * time.Second
+	programDoesNotExistErr = "does not exist"
 )
 
 // ReconcilerCommon provides a skeleton for all *Program Reconcilers.
@@ -622,7 +624,7 @@ func (r *ReconcilerCommon) handleProgDelete(
 			opts := client.DeleteOptions{}
 			r.Logger.Info("Deleting bpfProgram", "Name", bpfProgram.Name, "Owner", bpfProgram.GetName())
 			if err := r.Delete(ctx, &bpfProgram, &opts); err != nil {
-				return internal.Requeue, fmt.Errorf("failed to create bpfProgram object: %v", err)
+				return internal.Requeue, fmt.Errorf("failed to delete bpfProgram object: %v", err)
 			}
 			return internal.Updated, nil
 		}
@@ -633,6 +635,43 @@ func (r *ReconcilerCommon) handleProgDelete(
 	return internal.Unchanged, nil
 }
 
+// unLoadAndDeleteProgramsList unloads and deletes BbpPrograms when the owning
+// *Program or BpfApplication is not being deleted itself, but something
+// has changed such that the BpfPrograms are no longer needed.
+func (r *ReconcilerCommon) unLoadAndDeleteBpfProgramsList(ctx context.Context, bpfProgramsList *bpfmaniov1alpha1.BpfProgramList, finalizerString string) (reconcile.Result, error) {
+	for _, bpfProgram := range bpfProgramsList.Items {
+		r.Logger.V(1).Info("Deleting bpfProgram", "Name", bpfProgram.Name)
+		id, err := bpfmanagentinternal.GetID(&bpfProgram)
+		if err != nil {
+			r.Logger.Error(err, "Failed to get bpf program ID")
+			return ctrl.Result{}, nil
+		}
+		r.Logger.Info("Calling bpfman to unload program on node", "bpfProgram Name", bpfProgram.Name, "Program ID", id)
+		if err := bpfmanagentinternal.UnloadBpfmanProgram(ctx, r.BpfmanClient, *id); err != nil {
+			if strings.Contains(err.Error(), programDoesNotExistErr) {
+				r.Logger.Info("Program not found on node", "bpfProgram Name", bpfProgram.Name, "Program ID", id)
+			} else {
+				r.Logger.Error(err, "Failed to unload Program")
+				return ctrl.Result{RequeueAfter: retryDurationAgent}, nil
+			}
+		}
+
+		if r.removeFinalizer(ctx, &bpfProgram, finalizerString) {
+			return ctrl.Result{}, nil
+		}
+
+		opts := client.DeleteOptions{}
+		r.Logger.Info("Deleting bpfProgram", "Name", bpfProgram.Name, "Owner", bpfProgram.GetName())
+		if err := r.Delete(ctx, &bpfProgram, &opts); err != nil {
+			return ctrl.Result{RequeueAfter: retryDurationAgent}, fmt.Errorf("failed to delete bpfProgram object: %v", err)
+		} else {
+			// we will deal one program at a time, so we can break out of the loop
+			break
+		}
+	}
+	return ctrl.Result{}, nil
+}
+
 // handleProgCreateOrUpdate compares the expected bpfPrograms to the existing
 // bpfPrograms.  If a bpfProgram is expected but doesn't exist, it is created.
 // If an expected bpfProgram exists, it is reconciled. If a bpfProgram exists