Add logic to register pod Info CRD

Currently tetragon uses cilium Endoints to get Pod Labels. Pod Info CRD
is a mapping between Pod IPs and Pod metadata therefore, instead of
using cilium endpoints, it can use the pod info CRs, hence removing the
dependency on cilium.

This commit:
- Added flag to skip PodInfo CRD in the operator, tetragon will use
  endpoints in this case.
- Integrated  logic to register the PodInfo CRD with the API server,
  similar to tracing Policies CRD.

Signed-off-by: Prateek Singh <prateeksingh9741@gmail.com>
Signed-off-by: Michi Mutsuzaki <michi@isovalent.com>

install/kubernetes/templates/clusterrole.yaml

@@ -45,7 +45,7 @@ rules:
     resourceNames:
       - tracingpolicies.cilium.io
       - tracingpoliciesnamespaced.cilium.io
-      - tetragonpods.cilium.io
+      - podinfo.cilium.io
     verbs:
       - update
       - get

install/kubernetes/templates/operator_clusterrole.yaml

@@ -17,7 +17,7 @@ rules:
   - apiGroups:
       - cilium.io
     resources:
-      - tetragonpods
+      - podinfo
     verbs:
       - create
       - delete

operator/crd/crd.go

@@ -55,8 +55,9 @@ func RegisterCRDs() {
 	// Register the CRDs after validating that we are running on a supported
 	// version of K8s.
 	if !option.Config.SkipCRDCreation {
-		if err := client.RegisterCRDs(k8sAPIExtClient); err != nil {
-			log.WithError(err).Fatal("Unable to register CRDs")
+		// if skipPodInfoCRD flag set true, don't register Pod Info CRD.
+		if err := client.RegisterCRDs(k8sAPIExtClient, option.Config.SkipPodInfoCRD); err != nil {
+			log.WithError(err).Fatal("Unable to Register CRDs")
 		}
 	} else {
 		log.Info("Skipping creation of CRDs")

operator/flags.go

@@ -45,12 +45,15 @@ func initializeFlags() {
 
 	flags.String(operatorOption.ConfigDir, "", "Directory in which tetragon-operator-config configmap is mounted")
 
+	flags.Bool(operatorOption.SkipPodInfoCRD, false, "When true, PodInfo Custom Resource Definition (CRD) will not be created")
+
 	viper.BindPFlags(flags)
 }
 
-// Populate sets all options with the values from viper.
+// configPopulate sets all options with the values from viper.
 func configPopulate() {
 	operatorOption.Config.SkipCRDCreation = viper.GetBool(operatorOption.SkipCRDCreation)
 	operatorOption.Config.KubeCfgPath = viper.GetString(operatorOption.KubeCfgPath)
 	operatorOption.Config.ConfigDir = viper.GetString(operatorOption.ConfigDir)
+	operatorOption.Config.SkipPodInfoCRD = viper.GetBool(operatorOption.SkipPodInfoCRD)
 }

operator/option/config.go

@@ -18,6 +18,10 @@ const (
 
 	// ConfigDir specifies the directory in which tetragon-operator-config configmap is mounted.
 	ConfigDir = "config-dir"
+
+	// SkipPodInfoCRD specifies whether the tetragonPod CustomResourceDefinition will be
+	// disabled
+	SkipPodInfoCRD = "skip-pod-info-crd"
 )
 
 // OperatorConfig is the configuration used by the operator.
@@ -31,6 +35,9 @@ type OperatorConfig struct {
 
 	// ConfigDir specifies the directory in which tetragon-operator-config configmap is mounted.
 	ConfigDir string
+
+	// SkipPodInfoCRD disables creation of the TetragonPod CustomResourceDefinition only.
+	SkipPodInfoCRD bool
 }
 
 // Config represents the operator configuration.

pkg/k8s/Makefile

@@ -29,6 +29,7 @@ generate:
 	  -o . \
 	  --trim-path-prefix github.com/cilium/tetragon/pkg/k8s \
 	  --plural-exceptions TracingPolicyNamespaced:TracingPoliciesNamespaced \
+	  --plural-exceptions PodInfo:PodInfo \
 
 .PHONY: vendor
 vendor:

pkg/k8s/apis/cilium.io/client/crds/v1alpha1/cilium.io_podinfo.yaml

@@ -0,0 +1,70 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.12.1
+  creationTimestamp: null
+  name: podinfo.cilium.io
+spec:
+  group: cilium.io
+  names:
+    kind: PodInfo
+    listKind: PodInfoList
+    plural: podinfo
+    singular: podinfo
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: PodInfo is the Scheme for the Podinfo API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              hostNetwork:
+                description: Host networking requested for this pod. Use the host's
+                  network namespace. If this option is set, the ports that will be
+                  used must be specified.
+                type: boolean
+            type: object
+          status:
+            properties:
+              podIP:
+                description: IP address allocated to the pod. Routable at least within
+                  the cluster. Empty if not yet allocated.
+                type: string
+              podIPs:
+                description: List of Ip addresses allocated to the pod. 0th entry
+                  must be same as PodIP.
+                items:
+                  properties:
+                    IP:
+                      description: IP is an IP address (IPv4 or IPv6) assigned to
+                        the pod
+                      type: string
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

pkg/k8s/apis/cilium.io/client/register_v1_crd.go

@@ -45,11 +45,11 @@ var (
 
 // CreateCustomResourceDefinitions creates our CRD objects in the Kubernetes
 // cluster.
-func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface) error {
+func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
 	g, _ := errgroup.WithContext(context.Background())
 
 	g.Go(func() error {
-		return createTPCRDs(clientset)
+		return createCRDs(clientset, skipPodInfo)
 	})
 
 	return g.Wait()
@@ -61,10 +61,13 @@ var (
 
 	//go:embed crds/v1alpha1/cilium.io_tracingpoliciesnamespaced.yaml
 	crdsv1Alpha1TracingPoliciesNamespaced []byte
+
+	//go:embed crds/v1alpha1/cilium.io_podinfo.yaml
+	crdsv1Alpha1PodInfo []byte
 )
 
-// GetPregeneratedCRD returns the pregenerated CRD based on the requested CRD
-// name. The pregenerated CRDs are generated by the controller-gen tool and
+// GetPregeneratedCRD returns the pre-generated CRD based on the requested CRD
+// name. The pre-generated CRDs are generated by the controller-gen tool and
 // serialized into binary form by go-bindata. This function retrieves CRDs from
 // the binary form.
 func GetPregeneratedCRD(crdName string) apiextensionsv1.CustomResourceDefinition {
@@ -76,46 +79,60 @@ func GetPregeneratedCRD(crdName string) apiextensionsv1.CustomResourceDefinition
 	scopedLog := log.WithField("crdName", crdName)
 
 	switch crdName {
+	// Tracing Policy Custom Resource Definition.
 	case v1alpha1.TPCRDName:
 		crdBytes = crdsv1Alpha1TracingPolicies
+	// Namespaced Tracing Policy Custom Resource Definition.
 	case v1alpha1.TPNamespacedCRDName:
 		crdBytes = crdsv1Alpha1TracingPoliciesNamespaced
+	// Pod Info Custom Resource Definition.
+	case v1alpha1.PICRDName:
+		crdBytes = crdsv1Alpha1PodInfo
 	default:
-		scopedLog.Fatal("Pregenerated CRD does not exist")
+		scopedLog.Fatal("Pre-generated CRD does not exist")
 	}
 
 	isoCRD := apiextensionsv1.CustomResourceDefinition{}
 	err = yaml.Unmarshal(crdBytes, &isoCRD)
 	if err != nil {
-		scopedLog.WithError(err).Fatal("Error unmarshalling pregenerated CRD")
+		scopedLog.WithError(err).Fatal("Error unmarshalling pre-generated CRD")
 	}
 
 	return isoCRD
 }
 
-func createTPCRDs(clientset apiextensionsclient.Interface) error {
-	// custer-wide tracing policy CRD
-	isoCRD := GetPregeneratedCRD(v1alpha1.TPCRDName)
-	if err := createUpdateCRD(
-		clientset,
-		v1alpha1.TPCRDName,
-		constructV1CRD(v1alpha1.TPName, isoCRD),
-		newDefaultPoller(),
-	); err != nil {
-		return err
+// createTPCRDs creates or updates the CRDs with the API server.
+func createCRDs(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
+
+	doCreateCRD := func(crdName, name string) error {
+		isoCRD := GetPregeneratedCRD(crdName)
+		err := createUpdateCRD(
+			clientset,
+			crdName,
+			constructV1CRD(name, isoCRD),
+			newDefaultPoller())
+		if err != nil {
+			err = fmt.Errorf("failed to create %s: %w", crdName, err)
+			return err
+		}
+		return nil
 	}
 
-	// namespaced tracing policy CRD
-	isoCRD = GetPregeneratedCRD(v1alpha1.TPNamespacedCRDName)
-	if err := createUpdateCRD(
-		clientset,
-		v1alpha1.TPNamespacedCRDName,
-		constructV1CRD(v1alpha1.TPNamespacedName, isoCRD),
-		newDefaultPoller(),
-	); err != nil {
-		return err
+	var ret error
+	// Create TracingPolicy Custom Resource Definition
+	ret = goerrors.Join(ret, doCreateCRD(v1alpha1.TPCRDName, v1alpha1.TPName))
+
+	// Create TracingPoliciesNamespaced Custom Resource Definition
+	ret = goerrors.Join(ret, doCreateCRD(v1alpha1.TPNamespacedCRDName, v1alpha1.TPNamespacedName))
+
+	if skipPodInfo {
+		return ret
 	}
-	return nil
+
+	// Create the PodInfo Custom Resource Definition.
+	ret = goerrors.Join(ret, doCreateCRD(v1alpha1.PICRDName, v1alpha1.PIName))
+
+	return ret
 }
 
 // createUpdateCRD ensures the CRD object is installed into the K8s cluster. It
@@ -145,10 +162,12 @@ func createUpdateCRD(
 	}
 
 	v1CRDClient := clientset.ApiextensionsV1()
+	// get the CRD if it is already registered.
 	clusterCRD, err := v1CRDClient.CustomResourceDefinitions().Get(
 		context.TODO(),
 		crd.ObjectMeta.Name,
 		metav1.GetOptions{})
+	// If not found, register the CRD.
 	if errors.IsNotFound(err) {
 		scopedLog.Info("Creating CRD (CustomResourceDefinition)...")
 
@@ -162,10 +181,12 @@ func createUpdateCRD(
 			return nil
 		}
 	}
+	// some other error occurred while getting the CRD from the API server.
 	if err != nil {
 		return err
 	}
 
+	// CRD already registered, update it with the new version.
 	if err := updateV1CRD(scopedLog, crd, clusterCRD, v1CRDClient, poller); err != nil {
 		return err
 	}
@@ -178,6 +199,7 @@ func createUpdateCRD(
 	return nil
 }
 
+// constructV1CRD creates the CRD to be registered.
 func constructV1CRD(
 	name string,
 	template apiextensionsv1.CustomResourceDefinition,
@@ -203,6 +225,10 @@ func constructV1CRD(
 	}
 }
 
+// needsUpdateV1 returns true if the CRD needs to be updated, in three cases:
+// - CRD does not have a Schema.
+// - CRD does not have labels, equal to the Schema Version Key.
+// - Schema Version Key of the CRD has changed.
 func needsUpdateV1(clusterCRD *apiextensionsv1.CustomResourceDefinition) bool {
 	if clusterCRD.Spec.Versions[0].Schema == nil {
 		// no validation detected
@@ -223,6 +249,7 @@ func needsUpdateV1(clusterCRD *apiextensionsv1.CustomResourceDefinition) bool {
 	return false
 }
 
+// updateV1CRD checks and updates the pre-existing CRD with the new one.
 func updateV1CRD(
 	scopedLog *logrus.Entry,
 	crd, clusterCRD *apiextensionsv1.CustomResourceDefinition,
@@ -245,7 +272,7 @@ func updateV1CRD(
 				return false, err
 			}
 
-			// This seems too permissive but we only get here if the version is
+			// This seems too permissive, but we only get here if the version is
 			// different per needsUpdate above. If so, we want to update on any
 			// validation change including adding or removing validation.
 			if needsUpdateV1(clusterCRD) {
@@ -350,10 +377,10 @@ func (p defaultPoll) Poll(
 	return wait.Poll(interval, duration, conditionFn)
 }
 
-// RegisterCRDs registers all CRDs with the K8s apiserver.
-func RegisterCRDs(clientset apiextensionsclient.Interface) error {
-	if err := CreateCustomResourceDefinitions(clientset); err != nil {
-		return fmt.Errorf("Unable to create custom resource definition: %w", err)
+// RegisterCRDs registers all CRDs with the K8s api server.
+func RegisterCRDs(clientset apiextensionsclient.Interface, skipPodInfo bool) error {
+	if err := CreateCustomResourceDefinitions(clientset, skipPodInfo); err != nil {
+		return fmt.Errorf("unable to create custom resource definition: %w", err)
 	}
 
 	return nil

pkg/k8s/apis/cilium.io/v1alpha1/register.go

@@ -24,6 +24,9 @@ const (
 
 	// TPNamespacedCRDName is the full name of the TracingPolicy CRD.
 	TPNamespacedCRDName = TPNamespacedKindDefinition + "/" + CRDVersion
+
+	// PICRDName is the full name of the Tetragon Pod Info CRD.
+	PICRDName = PIKindDefinition + "/" + CRDVersion
 )
 
 // SchemeGroupVersion is group version used to register these objects
@@ -55,6 +58,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&TracingPolicyList{},
 		&TracingPolicyNamespaced{},
 		&TracingPolicyNamespacedList{},
+		&PodInfo{},
+		&PodInfoList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil