Avoid reconciling cluster-wide resources in odiglet (#1355)

Co-authored-by: Amir Blum <amirgiraffe@gmail.com>

api/config/crd/bases/odigos.io_instrumentationconfigs.yaml

@@ -109,6 +109,10 @@ spec:
                   - optionKey
                   type: object
                 type: array
+              runtimeDetailsInvalidated:
+                description: true when the runtime details are invalidated and should
+                  be recalculated
+                type: boolean
               sdkConfigs:
                 description: |-
                   Configuration for the OpenTelemetry SDKs that this workload should use.
@@ -157,9 +161,6 @@ spec:
                   - language
                   type: object
                 type: array
-            required:
-            - config
-            - sdkConfigs
             type: object
         type: object
     served: true

api/odigos/v1alpha1/instrumentatioconfig_types.go

@@ -19,15 +19,18 @@ type InstrumentationConfig struct {
 // Config for the OpenTelemeetry SDKs that should be applied to a workload.
 // The workload is identified by the owner reference
 type InstrumentationConfigSpec struct {
+	// true when the runtime details are invalidated and should be recalculated
+	RuntimeDetailsInvalidated bool `json:"runtimeDetailsInvalidated,omitempty"`
+
 	// config for this workload.
 	// the config is a list to allow for multiple config options and values to be applied.
 	// the list is processed in order, and the first matching config is applied.
-	Config []WorkloadInstrumentationConfig `json:"config"`
+	Config []WorkloadInstrumentationConfig `json:"config,omitempty"`
 
 	// Configuration for the OpenTelemetry SDKs that this workload should use.
 	// The SDKs are identified by the programming language they are written in.
 	// TODO: consider adding more granular control over the SDKs, such as community/enterprise, native/ebpf.
-	SdkConfigs []SdkConfig `json:"sdkConfigs"`
+	SdkConfigs []SdkConfig `json:"sdkConfigs,omitempty"`
 }
 
 type SdkConfig struct {

cli/cmd/resources/instrumentor.go

@@ -347,6 +347,21 @@ func NewInstrumentorClusterRole() *rbacv1.ClusterRole {
 					"destinations/status",
 				},
 			},
+			{
+				Verbs: []string{
+					"create",
+					"delete",
+					"get",
+					"list",
+					"patch",
+					"update",
+					"watch",
+				},
+				APIGroups: []string{"odigos.io"},
+				Resources: []string{
+					"instrumentationconfigs",
+				},
+			},
 		},
 	}
 }

cli/cmd/resources/odiglet.go

@@ -259,6 +259,8 @@ func NewOdigletClusterRole(psp bool) *rbacv1.ClusterRole {
 					"get",
 					"list",
 					"watch",
+					"patch",
+					"update",
 				},
 				APIGroups: []string{"odigos.io"},
 				Resources: []string{
@@ -479,10 +481,18 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 							},
 						},
 						{
-							Name: "var-dir",
+							Name: "pod-resources",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var",
+									Path: "/var/lib/kubelet/pod-resources",
+								},
+							},
+						},
+						{
+							Name: "device-plugins-dir",
+							VolumeSource: corev1.VolumeSource{
+								HostPath: &corev1.HostPathVolumeSource{
+									Path: "/var/lib/kubelet/device-plugins",
 								},
 							},
 						},
@@ -503,6 +513,26 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 							},
 						},
 					}, odigosSeLinuxHostVolumes...),
+					InitContainers: []corev1.Container{
+						{
+							Name:  "init",
+							Image: containers.GetImageName(imagePrefix, imageName, version),
+							Command: []string{
+								"/root/odiglet",
+							},
+							Args: []string{
+								"init",
+							},
+							Resources: corev1.ResourceRequirements{},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "odigos",
+									MountPath: "/var/odigos",
+								},
+							},
+							ImagePullPolicy: "IfNotPresent",
+						},
+					},
 					Containers: []corev1.Container{
 						{
 							Name:  OdigletContainerName,
@@ -533,7 +563,7 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 									},
 								},
 								{
-									Name: "OTEL_LOG_LEVEL",
+									Name:  "OTEL_LOG_LEVEL",
 									Value: "info",
 								},
 							}, dynamicEnv...),
@@ -549,19 +579,22 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 							Resources: corev1.ResourceRequirements{},
 							VolumeMounts: append([]corev1.VolumeMount{
 								{
-									Name:             "run-dir",
-									MountPath:        "/run",
-									MountPropagation: ptrMountPropagationMode("Bidirectional"),
+									Name:      "run-dir",
+									MountPath: "/run",
+								},
+								{
+									Name:      "device-plugins-dir",
+									MountPath: "/var/lib/kubelet/device-plugins",
 								},
 								{
-									Name:             "var-dir",
-									MountPath:        "/var",
-									MountPropagation: ptrMountPropagationMode("Bidirectional"),
+									Name:      "pod-resources",
+									MountPath: "/var/lib/kubelet/pod-resources",
+									ReadOnly:  true,
 								},
 								{
-									Name:             "odigos",
-									MountPath:        "/var/odigos",
-									MountPropagation: ptrMountPropagationMode("Bidirectional"),
+									Name:      "odigos",
+									MountPath: "/var/odigos",
+									ReadOnly:  true,
 								},
 								{
 									Name:      "kernel-debug",

instrumentor/controllers/deleteinstrumentedapplication/common.go

@@ -3,12 +3,9 @@ package deleteinstrumentedapplication
 import (
 	"context"
 
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-
 	odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
 	"github.com/odigos-io/odigos/common/consts"
 	"github.com/odigos-io/odigos/k8sutils/pkg/workload"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -17,7 +14,7 @@ import (
 
 func reconcileWorkloadObject(ctx context.Context, kubeClient client.Client, workloadObject client.Object) error {
 	logger := log.FromContext(ctx)
-	instEffectiveEnabled, err := isWorkloadInstrumentationEffectiveEnabled(ctx, kubeClient, workloadObject)
+	instEffectiveEnabled, err := workload.IsWorkloadInstrumentationEffectiveEnabled(ctx, kubeClient, workloadObject)
 	if err != nil {
 		logger.Error(err, "error checking if instrumentation is effective")
 		return err
@@ -47,50 +44,30 @@ func deleteWorkloadInstrumentedApplication(ctx context.Context, kubeClient clien
 	kind := workload.GetWorkloadKind(workloadObject)
 	instrumentedApplicationName := workload.GetRuntimeObjectName(name, kind)
 
-	err := kubeClient.Delete(ctx, &odigosv1.InstrumentedApplication{
+	instAppErr := kubeClient.Delete(ctx, &odigosv1.InstrumentedApplication{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: ns,
 			Name:      instrumentedApplicationName,
 		},
 	})
-	if err != nil {
-		return client.IgnoreNotFound(err)
-	}
 
-	logger := log.FromContext(ctx)
-	logger.V(1).Info("instrumented application deleted", "namespace", ns, "name", name, "kind", kind)
-	return nil
-}
-
-func isWorkloadInstrumentationEffectiveEnabled(ctx context.Context, kubeClient client.Client, obj client.Object) (bool, error) {
-
-	// if the object itself is labeled, we will use that value
-	workloadLabels := obj.GetLabels()
-	if val, exists := workloadLabels[consts.OdigosInstrumentationLabel]; exists {
-		return val == consts.InstrumentationEnabled, nil
+	instConfigErr := kubeClient.Delete(ctx, &odigosv1.InstrumentationConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      instrumentedApplicationName,
+		},
+	})
+	if instAppErr != nil {
+		return client.IgnoreNotFound(instAppErr)
 	}
 
-	// we will get here if the workload instrumentation label is not set.
-	// no label means inherit the instrumentation value from namespace.
-	var ns corev1.Namespace
-	err := kubeClient.Get(ctx, client.ObjectKey{Name: obj.GetNamespace()}, &ns)
-	if err != nil {
-		logger := log.FromContext(ctx)
-		if apierrors.IsNotFound(err) {
-			logger.V(1).Info("namespace not found")
-			return false, nil
-		}
-
-		logger.Error(err, "error fetching namespace object")
-		return false, err
+	if instConfigErr != nil {
+		return client.IgnoreNotFound(instConfigErr)
 	}
 
-	return isInstrumentationLabelEnabled(&ns), nil
-}
-
-func isInstrumentationLabelEnabled(workloadObject client.Object) bool {
-	labels := workloadObject.GetLabels()
-	return labels[consts.OdigosInstrumentationLabel] == consts.InstrumentationEnabled
+	logger := log.FromContext(ctx)
+	logger.V(1).Info("instrumented application deleted", "namespace", ns, "name", name, "kind", kind)
+	return nil
 }
 
 func removeReportedNameAnnotation(ctx context.Context, kubeClient client.Client, workloadObject client.Object) error {

instrumentor/controllers/deleteinstrumentedapplication/instrumentedapplication_controller.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/odigos-io/odigos/k8sutils/pkg/workload"
+
 	odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -82,7 +84,7 @@ func (r *InstrumentedApplicationReconciler) Reconcile(ctx context.Context, req c
 		return ctrl.Result{}, err
 	}
 
-	instEffectiveEnabled, err := isWorkloadInstrumentationEffectiveEnabled(ctx, r.Client, workloadObject)
+	instEffectiveEnabled, err := workload.IsWorkloadInstrumentationEffectiveEnabled(ctx, r.Client, workloadObject)
 	if err != nil {
 		logger.Error(err, "error checking if instrumentation is effective")
 		return ctrl.Result{}, err

instrumentor/controllers/deleteinstrumentedapplication/manager.go

@@ -2,7 +2,6 @@ package deleteinstrumentedapplication
 
 import (
 	odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
-	k8sutils "github.com/odigos-io/odigos/k8sutils/pkg/client"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -11,17 +10,12 @@ import (
 )
 
 func SetupWithManager(mgr ctrl.Manager) error {
-	// Create a new client with fallback to API server
-	// We are doing this because client-go cache is not supporting dynamic cache rules
-	// Sometimes we will need to get/list objects that are out of the cache (e.g. when namespace is labeled)
-	clientWithFallback := k8sutils.NewKubernetesClientFromCacheWithAPIFallback(mgr.GetClient(), mgr.GetAPIReader())
-
 	err := builder.
 		ControllerManagedBy(mgr).
 		For(&appsv1.Deployment{}).
 		WithEventFilter(predicate.LabelChangedPredicate{}).
 		Complete(&DeploymentReconciler{
-			Client: clientWithFallback,
+			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		})
 	if err != nil {
@@ -33,7 +27,7 @@ func SetupWithManager(mgr ctrl.Manager) error {
 		For(&appsv1.StatefulSet{}).
 		WithEventFilter(predicate.LabelChangedPredicate{}).
 		Complete(&StatefulSetReconciler{
-			Client: clientWithFallback,
+			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		})
 	if err != nil {
@@ -45,7 +39,7 @@ func SetupWithManager(mgr ctrl.Manager) error {
 		For(&appsv1.DaemonSet{}).
 		WithEventFilter(predicate.LabelChangedPredicate{}).
 		Complete(&DaemonSetReconciler{
-			Client: clientWithFallback,
+			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		})
 	if err != nil {
@@ -57,7 +51,7 @@ func SetupWithManager(mgr ctrl.Manager) error {
 		For(&corev1.Namespace{}).
 		WithEventFilter(predicate.LabelChangedPredicate{}).
 		Complete(&NamespaceReconciler{
-			Client: clientWithFallback,
+			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		})
 	if err != nil {
@@ -68,7 +62,7 @@ func SetupWithManager(mgr ctrl.Manager) error {
 		ControllerManagedBy(mgr).
 		For(&odigosv1.InstrumentedApplication{}).
 		Complete(&InstrumentedApplicationReconciler{
-			Client: clientWithFallback,
+			Client: mgr.GetClient(),
 			Scheme: mgr.GetScheme(),
 		})
 	if err != nil {

instrumentor/controllers/deleteinstrumentedapplication/namespace_controller.go

@@ -19,9 +19,9 @@ package deleteinstrumentedapplication
 import (
 	"context"
 
-	appsv1 "k8s.io/api/apps/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"github.com/odigos-io/odigos/k8sutils/pkg/workload"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -40,20 +40,17 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	var ns corev1.Namespace
 	err := r.Get(ctx, client.ObjectKey{Name: req.Name}, &ns)
-	if err != nil {
-		if apierrors.IsNotFound(err) {
-			return ctrl.Result{}, nil
-		}
-
+	if client.IgnoreNotFound(err) != nil {
 		logger.Error(err, "error fetching namespace object")
 		return ctrl.Result{}, err
 	}
 
 	// If namespace is labeled, skip
-	if isInstrumentationLabelEnabled(&ns) {
+	if err == nil && workload.IsObjectLabeledForInstrumentation(&ns) {
 		return ctrl.Result{}, nil
 	}
 
+	// Because of cache settings in the controller, when namespace is unlabelled, it is appearing as not found
 	var deps appsv1.DeploymentList
 	err = r.Client.List(ctx, &deps, client.InNamespace(req.Name))
 	if client.IgnoreNotFound(err) != nil {
@@ -62,7 +59,7 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	for _, dep := range deps.Items {
-		if !isInstrumentationLabelEnabled(&dep) {
+		if !workload.IsObjectLabeledForInstrumentation(&dep) {
 			if err := deleteWorkloadInstrumentedApplication(ctx, r.Client, &dep); err != nil {
 				logger.Error(err, "error removing runtime details")
 				return ctrl.Result{}, err
@@ -83,7 +80,7 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	for _, s := range ss.Items {
-		if !isInstrumentationLabelEnabled(&s) {
+		if !workload.IsObjectLabeledForInstrumentation(&s) {
 			if err := deleteWorkloadInstrumentedApplication(ctx, r.Client, &s); err != nil {
 				logger.Error(err, "error removing runtime details")
 				return ctrl.Result{}, err
@@ -104,7 +101,7 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	for _, d := range ds.Items {
-		if !isInstrumentationLabelEnabled(&d) {
+		if !workload.IsObjectLabeledForInstrumentation(&d) {
 			if err := deleteWorkloadInstrumentedApplication(ctx, r.Client, &d); err != nil {
 				logger.Error(err, "error removing runtime details")
 				return ctrl.Result{}, err