Add logic to register pod Info CRD:

Currently tetragon uses cilium Endoints to get Pod Labels. Pod Info CRD is a mapping between Pod IPs and Pod metadata therefore, instead of using cilium endpoints, it can use the pod info CRs, hence removing the dependency on cilium.

This commit:
- Added flag to skip PodInfo CRD in the operator, tetragon will use endpoints in this case.
- Integrated  logic to register the PodInfo CRD with the API server, similar to tracing Policies CRD.

Signed-off-by: Prateek Singh <prateeksingh9741@gmail.com>

install/kubernetes/templates/clusterrole.yaml

@@ -45,7 +45,7 @@ rules:
     resourceNames:
       - tracingpolicies.cilium.io
       - tracingpoliciesnamespaced.cilium.io
-      - tetragonpods.cilium.io
+      - podinfo.cilium.io
     verbs:
       - update
       - get

install/kubernetes/templates/operator_clusterrole.yaml

@@ -17,7 +17,7 @@ rules:
   - apiGroups:
       - cilium.io
     resources:
-      - tetragonpods
+      - podinfo
     verbs:
       - create
       - delete

operator/crd/crd.go

@@ -55,8 +55,20 @@ func RegisterCRDs() {
 	// Register the CRDs after validating that we are running on a supported
 	// version of K8s.
 	if !option.Config.SkipCRDCreation {
-		if err := client.RegisterCRDs(k8sAPIExtClient); err != nil {
-			log.WithError(err).Fatal("Unable to register CRDs")
+		skipPodInfo := option.Config.SkipCRDCreation
+
+		// if skipTetragonPodCRD flag set true, don't register Pod Info CRD.
+		if skipPodInfo {
+			log.Info("Registering Tracing Policy CRD")
+			if err := client.RegisterCRDs(k8sAPIExtClient, skipPodInfo); err != nil {
+				log.WithError(err).Fatal("Unable to register CRDs")
+			}
+		} else {
+			log.Info("Registering Tracing Policy and PodInfo CRDs")
+			// Register TetragonPod CRD
+			if err := client.RegisterCRDs(k8sAPIExtClient, skipPodInfo); err != nil {
+				log.WithError(err).Fatal("Unable to register TetragonPod CRDs")
+			}
 		}
 	} else {
 		log.Info("Skipping creation of CRDs")

operator/flags.go

@@ -56,12 +56,15 @@ func initializeFlags() {
 
 	flags.String(operatorOption.ConfigDir, "", "Directory in which tetragon-operator-config configmap is mounted")
 
+	flags.Bool(operatorOption.SkipTetragonPodCRD, false, "When true, TetragonPod Custom Resource Definition (CRD) will not be created")
+
 	viper.BindPFlags(flags)
 }
 
-// Populate sets all options with the values from viper.
+// configPopulate sets all options with the values from viper.
 func configPopulate() {
 	operatorOption.Config.SkipCRDCreation = viper.GetBool(operatorOption.SkipCRDCreation)
 	operatorOption.Config.KubeCfgPath = viper.GetString(operatorOption.KubeCfgPath)
 	operatorOption.Config.ConfigDir = viper.GetString(operatorOption.ConfigDir)
+	operatorOption.Config.SkipTetragonPodCRD = viper.GetBool(operatorOption.SkipTetragonPodCRD)
 }

operator/option/config.go

@@ -29,6 +29,10 @@ const (
 
 	// ConfigDir specifies the directory in which tetragon-operator-config configmap is mounted.
 	ConfigDir = "config-dir"
+
+	// SkipTetragonPodCRD specifies whether the tetragonPod CustomResourceDefinition will be
+	// disabled
+	SkipTetragonPodCRD = "skip-tetragon-pod-crd"
 )
 
 // OperatorConfig is the configuration used by the operator.
@@ -42,6 +46,9 @@ type OperatorConfig struct {
 
 	// ConfigDir specifies the directory in which tetragon-operator-config configmap is mounted.
 	ConfigDir string
+
+	// SkipTetragonPodCRD disables creation of the TetragonPod CustomResourceDefinition only.
+	SkipTetragonPodCRD bool
 }
 
 // Config represents the operator configuration.

pkg/k8s/Makefile

@@ -29,6 +29,7 @@ generate:
 	  -o . \
 	  --trim-path-prefix github.com/cilium/tetragon/pkg/k8s \
 	  --plural-exceptions TracingPolicyNamespaced:TracingPoliciesNamespaced \
+	  --plural-exceptions PodInfo:PodInfo \
 
 .PHONY: vendor
 vendor:

pkg/k8s/apis/cilium.io/client/crds/v1alpha1/cilium.io_podinfo.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.11.3
+  creationTimestamp: null
+  name: podinfo.cilium.io
+spec:
+  group: cilium.io
+  names:
+    kind: PodInfo
+    listKind: PodInfoList
+    plural: podinfo
+    singular: podinfo
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: PodInfo is the Schema for the PodInfo API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              properties:
+                hostNetwork:
+                  description: Host networking requested for this pod. Use the host's
+                    network namespace. If this option is set, the ports that will be
+                    used must be specified.
+                  type: boolean
+              type: object
+            status:
+              properties:
+                podIP:
+                  description: IP address allocated to the pod. Routable at least within
+                    the cluster. Empty if not yet allocated.
+                  type: string
+                podIPs:
+                  description: List of Ip addresses allocated to the pod. 0th entry
+                    must be same as PodIP.
+                  items:
+                    properties:
+                      ip:
+                        description: ip is an IP address (IPv4 or IPv6) assigned to
+                          the pod
+                        type: string
+                    type: object
+                  type: array
+              type: object
+          type: object
+      served: true
+      storage: true

pkg/k8s/apis/cilium.io/client/crds/v1alpha1/cilium.io_podinfoes.yaml

@@ -0,0 +1,70 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.12.1
+  creationTimestamp: null
+  name: podinfoes.cilium.io
+spec:
+  group: cilium.io
+  names:
+    kind: PodInfo
+    listKind: PodInfoList
+    plural: podinfoes
+    singular: podinfo
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: PodInfo is the Scheme for the Podinfo API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              hostNetwork:
+                description: Host networking requested for this pod. Use the host's
+                  network namespace. If this option is set, the ports that will be
+                  used must be specified.
+                type: boolean
+            type: object
+          status:
+            properties:
+              podIP:
+                description: IP address allocated to the pod. Routable at least within
+                  the cluster. Empty if not yet allocated.
+                type: string
+              podIPs:
+                description: List of Ip addresses allocated to the pod. 0th entry
+                  must be same as PodIP.
+                items:
+                  properties:
+                    IP:
+                      description: IP is an IP address (IPv4 or IPv6) assigned to
+                        the pod
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

pkg/k8s/apis/cilium.io/client/register_v1_crd.go

@@ -45,11 +45,11 @@ var (
 
 // CreateCustomResourceDefinitions creates our CRD objects in the Kubernetes
 // cluster.
-func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface) error {
+func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
 	g, _ := errgroup.WithContext(context.Background())
 
 	g.Go(func() error {
-		return createTPCRDs(clientset)
+		return createTPCRDs(clientset, skipPodInfo)
 	})
 
 	return g.Wait()
@@ -61,10 +61,13 @@ var (
 
 	//go:embed crds/v1alpha1/cilium.io_tracingpoliciesnamespaced.yaml
 	crdsv1Alpha1TracingPoliciesNamespaced []byte
+
+	//go:embed crds/v1alpha1/cilium.io_podinfo.yaml
+	crdsv1Alpha1PodInfo []byte
 )
 
-// GetPregeneratedCRD returns the pregenerated CRD based on the requested CRD
-// name. The pregenerated CRDs are generated by the controller-gen tool and
+// GetPregeneratedCRD returns the pre-generated CRD based on the requested CRD
+// name. The pre-generated CRDs are generated by the controller-gen tool and
 // serialized into binary form by go-bindata. This function retrieves CRDs from
 // the binary form.
 func GetPregeneratedCRD(crdName string) apiextensionsv1.CustomResourceDefinition {
@@ -76,24 +79,30 @@ func GetPregeneratedCRD(crdName string) apiextensionsv1.CustomResourceDefinition
 	scopedLog := log.WithField("crdName", crdName)
 
 	switch crdName {
+	// Tracing Policy Custom Resource Definition.
 	case v1alpha1.TPCRDName:
 		crdBytes = crdsv1Alpha1TracingPolicies
+	// Namespaced Tracing Policy Custom Resource Definition.
 	case v1alpha1.TPNamespacedCRDName:
 		crdBytes = crdsv1Alpha1TracingPoliciesNamespaced
+	// Pod Info Custom Resource Definition.
+	case v1alpha1.PICRDName:
+		crdBytes = crdsv1Alpha1PodInfo
 	default:
-		scopedLog.Fatal("Pregenerated CRD does not exist")
+		scopedLog.Fatal("Pre-generated CRD does not exist")
 	}
 
 	isoCRD := apiextensionsv1.CustomResourceDefinition{}
 	err = yaml.Unmarshal(crdBytes, &isoCRD)
 	if err != nil {
-		scopedLog.WithError(err).Fatal("Error unmarshalling pregenerated CRD")
+		scopedLog.WithError(err).Fatal("Error unmarshalling pre-generated CRD")
 	}
 
 	return isoCRD
 }
 
-func createTPCRDs(clientset apiextensionsclient.Interface) error {
+// createTPCRDs creates or updates the CRDs with the API server.
+func createTPCRDs(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
 	// custer-wide tracing policy CRD
 	isoCRD := GetPregeneratedCRD(v1alpha1.TPCRDName)
 	if err := createUpdateCRD(
@@ -115,6 +124,19 @@ func createTPCRDs(clientset apiextensionsclient.Interface) error {
 	); err != nil {
 		return err
 	}
+
+	if !skipPodInfo {
+		isoCRD = GetPregeneratedCRD(v1alpha1.PICRDName)
+		if err := createUpdateCRD(
+			clientset,
+			v1alpha1.PICRDName,
+			constructV1CRD(v1alpha1.PIName, isoCRD),
+			newDefaultPoller(),
+		); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -145,10 +167,12 @@ func createUpdateCRD(
 	}
 
 	v1CRDClient := clientset.ApiextensionsV1()
+	// get the CRD if it is already registered.
 	clusterCRD, err := v1CRDClient.CustomResourceDefinitions().Get(
 		context.TODO(),
 		crd.ObjectMeta.Name,
 		metav1.GetOptions{})
+	// If not found, register the CRD.
 	if errors.IsNotFound(err) {
 		scopedLog.Info("Creating CRD (CustomResourceDefinition)...")
 
@@ -162,10 +186,12 @@ func createUpdateCRD(
 			return nil
 		}
 	}
+	// some other error occurred while getting the CRD from the API server.
 	if err != nil {
 		return err
 	}
 
+	// CRD already registered, update it with the new version.
 	if err := updateV1CRD(scopedLog, crd, clusterCRD, v1CRDClient, poller); err != nil {
 		return err
 	}
@@ -178,6 +204,7 @@ func createUpdateCRD(
 	return nil
 }
 
+// constructV1CRD creates the CRD to be registered.
 func constructV1CRD(
 	name string,
 	template apiextensionsv1.CustomResourceDefinition,
@@ -203,6 +230,10 @@ func constructV1CRD(
 	}
 }
 
+// needsUpdateV1 returns true if the CRD needs to be updated, in three cases:
+// - CRD does not have a Schema.
+// - CRD does not have labels, equal to the Schema Version Key.
+// - Schema Version Key of the CRD has changed.
 func needsUpdateV1(clusterCRD *apiextensionsv1.CustomResourceDefinition) bool {
 	if clusterCRD.Spec.Versions[0].Schema == nil {
 		// no validation detected
@@ -223,6 +254,7 @@ func needsUpdateV1(clusterCRD *apiextensionsv1.CustomResourceDefinition) bool {
 	return false
 }
 
+// updateV1CRD checks and updates the pre-existing CRD with the new one.
 func updateV1CRD(
 	scopedLog *logrus.Entry,
 	crd, clusterCRD *apiextensionsv1.CustomResourceDefinition,
@@ -245,7 +277,7 @@ func updateV1CRD(
 				return false, err
 			}
 
-			// This seems too permissive but we only get here if the version is
+			// This seems too permissive, but we only get here if the version is
 			// different per needsUpdate above. If so, we want to update on any
 			// validation change including adding or removing validation.
 			if needsUpdateV1(clusterCRD) {
@@ -350,10 +382,10 @@ func (p defaultPoll) Poll(
 	return wait.Poll(interval, duration, conditionFn)
 }
 
-// RegisterCRDs registers all CRDs with the K8s apiserver.
-func RegisterCRDs(clientset apiextensionsclient.Interface) error {
-	if err := CreateCustomResourceDefinitions(clientset); err != nil {
-		return fmt.Errorf("Unable to create custom resource definition: %w", err)
+// RegisterCRDs registers all CRDs with the K8s api server.
+func RegisterCRDs(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
+	if err := CreateCustomResourceDefinitions(clientset, skipPodInfo); err != nil {
+		return fmt.Errorf("unable to create custom resource definition: %w", err)
 	}
 
 	return nil