Add flags to make it possible using k8s api without TP CRDs

Fixes: #1880

Signed-off-by: Alexey Olshanskiy <gh@aohoy.dev>

cmd/tetragon/main.go

@@ -71,6 +71,7 @@ import (
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apiextensionsinformer "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -356,50 +357,16 @@ func tetragonExecute() error {
 	var k8sWatcher watcher.K8sResourceWatcher
 	if option.Config.EnableK8s {
 		log.Info("Enabling Kubernetes API")
-		crds := map[string]struct{}{
-			v1alpha1.TPName:           {},
-			v1alpha1.TPNamespacedName: {},
-		}
-		if option.Config.EnablePodInfo {
-			crds[v1alpha1.PIName] = struct{}{}
-		}
 		config, err := k8sconf.K8sConfig()
 		if err != nil {
 			return err
 		}
-		log.WithField("crds", crds).Info("Waiting for required CRDs")
-		var wg sync.WaitGroup
-		wg.Add(1)
-		k8sClient := kubernetes.NewForConfigOrDie(config)
-		crdClient := apiextensionsclientset.NewForConfigOrDie(config)
-		crdInformer := apiextensionsinformer.NewCustomResourceDefinitionInformer(crdClient, 0*time.Second, nil)
-		_, err = crdInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
-			AddFunc: func(obj interface{}) {
-				crdObject, ok := obj.(*v1.CustomResourceDefinition)
-				if !ok {
-					log.WithField("obj", obj).Warn("Received an invalid object")
-					return
-				}
-				if _, ok := crds[crdObject.Name]; ok {
-					log.WithField("crd", crdObject.Name).Info("Found CRD")
-					delete(crds, crdObject.Name)
-					if len(crds) == 0 {
-						log.Info("Found all the required CRDs")
-						wg.Done()
-					}
-				}
-			},
-		})
-		if err != nil {
-			log.WithError(err).Error("failed to add event handler")
+
+		if err := waitCRDs(config); err != nil {
 			return err
 		}
-		stop := make(chan struct{})
-		go func() {
-			crdInformer.Run(stop)
-		}()
-		wg.Wait()
-		close(stop)
+
+		k8sClient := kubernetes.NewForConfigOrDie(config)
 		k8sWatcher = watcher.NewK8sWatcher(k8sClient, 60*time.Second)
 	} else {
 		log.Info("Disabling Kubernetes API")
@@ -451,7 +418,7 @@ func tetragonExecute() error {
 	log.WithField("enabled", option.Config.ExportFilename != "").WithField("fileName", option.Config.ExportFilename).Info("Exporter configuration")
 	obs.AddListener(pm)
 	saveInitInfo()
-	if option.Config.EnableK8s {
+	if option.Config.EnableK8s && option.Config.EnableTracingPolicyCRD {
 		go crd.WatchTracePolicy(ctx, observer.GetSensorManager())
 	}
 
@@ -495,6 +462,57 @@ func tetragonExecute() error {
 	return obs.Start(ctx)
 }
 
+func waitCRDs(config *rest.Config) error {
+	crds := make(map[string]struct{})
+
+	if option.Config.EnableTracingPolicyCRD {
+		crds[v1alpha1.TPName] = struct{}{}
+		crds[v1alpha1.TPNamespacedName] = struct{}{}
+	}
+	if option.Config.EnablePodInfo {
+		crds[v1alpha1.PIName] = struct{}{}
+	}
+
+	if len(crds) == 0 {
+		log.Info("No CRDs are enabled")
+		return nil
+	}
+
+	log.WithField("crds", crds).Info("Waiting for required CRDs")
+	var wg sync.WaitGroup
+	wg.Add(1)
+	crdClient := apiextensionsclientset.NewForConfigOrDie(config)
+	crdInformer := apiextensionsinformer.NewCustomResourceDefinitionInformer(crdClient, 0*time.Second, nil)
+	_, err := crdInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			crdObject, ok := obj.(*v1.CustomResourceDefinition)
+			if !ok {
+				log.WithField("obj", obj).Warn("Received an invalid object")
+				return
+			}
+			if _, ok := crds[crdObject.Name]; ok {
+				log.WithField("crd", crdObject.Name).Info("Found CRD")
+				delete(crds, crdObject.Name)
+				if len(crds) == 0 {
+					log.Info("Found all the required CRDs")
+					wg.Done()
+				}
+			}
+		},
+	})
+	if err != nil {
+		log.WithError(err).Error("failed to add event handler")
+		return err
+	}
+	stop := make(chan struct{})
+	go func() {
+		crdInformer.Run(stop)
+	}()
+	wg.Wait()
+	close(stop)
+	return nil
+}
+
 func loadTpFromDir(ctx context.Context, dir string) error {
 	tpMaxDepth := 1
 	tpFS := os.DirFS(dir)

install/kubernetes/tetragon/templates/operator_configmap.yaml

@@ -8,3 +8,4 @@ metadata:
 data:
   skip-crd-creation: {{ .Values.tetragonOperator.skipCRDCreation | quote }}
   skip-pod-info-crd: {{ not .Values.tetragonOperator.podInfo.enabled | quote }}
+  skip-tracing-policy-crd: {{ not .Values.tetragonOperator.tracingPolicy.enabled | quote }}

install/kubernetes/tetragon/templates/tetragon_configmap.yaml

@@ -56,4 +56,5 @@ data:
   enable-msg-handling-latency: "true"
 {{- end }}
   enable-pod-info: {{ .Values.tetragonOperator.podInfo.enabled | quote }}
+  enable-tracing-policy-crd: {{ .Values.tetragonOperator.tracingPolicy.enabled | quote }}
   {{- include "configmap.extra" . | nindent 2 }}

install/kubernetes/tetragon/values.yaml

@@ -232,6 +232,9 @@ tetragonOperator:
     # -- Enables the PodInfo CRD and the controller that reconciles PodInfo
     # custom resources.
     enabled: false
+  tracingPolicy:
+    # -- Enables the TracingPolicy and TracingPolicyNamespaced CRD creation.
+    enabled: true
   # -- Enables the Tetragon Operator metrics.
   prometheus:
     enabled: true

operator/cmd/common/common.go

@@ -21,6 +21,7 @@ func AddCommonFlags(cmd *cobra.Command) {
 	flags.String(operatorOption.KubeCfgPath, "", "Kubeconfig filepath to connect to k8s")
 	flags.String(operatorOption.ConfigDir, "", "Directory in which tetragon-operator-config configmap is mounted")
 	flags.Bool(operatorOption.SkipPodInfoCRD, false, "When true, PodInfo Custom Resource Definition (CRD) will not be created")
+	flags.Bool(operatorOption.SkipTracingPolicyCRD, false, "When true, TracingPolicy and TracingPolicyNamespaced Custom Resource Definition (CRD) will not be created")
 }
 
 func Initialize(cmd *cobra.Command) {

operator/crd/crd.go

@@ -58,7 +58,12 @@ func RegisterCRDs() {
 
 	crds := []crdutils.CRD{}
 	for _, crd := range client.AllCRDs {
-		if option.Config.SkipPodInfoCRD && crd.CRDName == client.PodInfoCRD.CRDName {
+		switch {
+		case option.Config.SkipPodInfoCRD && crd.CRDName == client.PodInfoCRD.CRDName:
+			continue
+		case option.Config.SkipTracingPolicyCRD && crd.CRDName == client.TracingPolicyCRD.CRDName:
+			continue
+		case option.Config.SkipTracingPolicyCRD && crd.CRDName == client.TracingPolicyNamespacedCRD.CRDName:
 			continue
 		}
 		crds = append(crds, crd)

operator/option/config.go

@@ -26,6 +26,10 @@ const (
 	// SkipPodInfoCRD specifies whether the tetragonPod CustomResourceDefinition will be
 	// disabled
 	SkipPodInfoCRD = "skip-pod-info-crd"
+
+	// SkipTracingPolicyCRD specifies whether the tracing-policies CustomResourceDefinition will be
+	// disabled
+	SkipTracingPolicyCRD = "skip-tracing-policy-crd"
 )
 
 // OperatorConfig is the configuration used by the operator.
@@ -42,6 +46,10 @@ type OperatorConfig struct {
 
 	// SkipPodInfoCRD disables creation of the TetragonPod CustomResourceDefinition only.
 	SkipPodInfoCRD bool
+
+	// SkipTracingPolicyCRD disables creation of the TracingPolicy and
+	// TracingPolicyNamespaced CustomResourceDefinition only.
+	SkipTracingPolicyCRD bool
 }
 
 // Config represents the operator configuration.
@@ -53,4 +61,5 @@ func ConfigPopulate() {
 	Config.KubeCfgPath = viper.GetString(KubeCfgPath)
 	Config.ConfigDir = viper.GetString(ConfigDir)
 	Config.SkipPodInfoCRD = viper.GetBool(SkipPodInfoCRD)
+	Config.SkipTracingPolicyCRD = viper.GetBool(SkipTracingPolicyCRD)
 }

pkg/option/config.go

@@ -83,7 +83,8 @@ type config struct {
 
 	KMods []string
 
-	EnablePodInfo bool
+	EnablePodInfo          bool
+	EnableTracingPolicyCRD bool
 
 	ExposeKernelAddresses bool
 }

pkg/option/flags.go

@@ -86,7 +86,8 @@ const (
 
 	KeyKmods = "kmods"
 
-	KeyEnablePodInfo = "enable-pod-info"
+	KeyEnablePodInfo          = "enable-pod-info"
+	KeyEnableTracingPolicyCRD = "enable-tracing-policy-crd"
 
 	KeyExposeKernelAddresses = "expose-kernel-addresses"
 )
@@ -163,6 +164,7 @@ func ReadAndSetFlags() error {
 	Config.KMods = viper.GetStringSlice(KeyKmods)
 
 	Config.EnablePodInfo = viper.GetBool(KeyEnablePodInfo)
+	Config.EnableTracingPolicyCRD = viper.GetBool(KeyEnableTracingPolicyCRD)
 
 	Config.TracingPolicy = viper.GetString(KeyTracingPolicy)
 
@@ -269,6 +271,7 @@ func AddFlags(flags *pflag.FlagSet) {
 	flags.String(KeyRBQueueSize, "65535", "Set size of channel between ring buffer and sensor go routines (default 65k, allows K/M/G suffix)")
 
 	flags.Bool(KeyEnablePodInfo, false, "Enable PodInfo custom resource")
+	flags.Bool(KeyEnableTracingPolicyCRD, true, "Enable TracingPolicy and TracingPolicyNamespaced custom resources")
 
 	flags.Bool(KeyExposeKernelAddresses, false, "Expose real kernel addresses in events stack traces")
 }