honeycomb: add support for metrics and logs (#38)

This PR adds metrics and logs support when sending data to Honeycomb.

DESTINATIONS.md

@@ -7,7 +7,7 @@
 | New Relic     | ✅     | ✅      | ✅   |
 | Datadog       | ✅     | ✅      |      |
 | Grafana Cloud | ✅     | ✅      | ✅   |
-| Honeycomb     | ✅     |         |      |
+| Honeycomb     | ✅     | ✅      | ✅   |
 | Logz.io       | ✅     | ✅      | ✅   |
 
 ## Open Source

autoscaler/controllers/datacollection/configmap.go

@@ -3,6 +3,7 @@ package datacollection
 import (
 	"context"
 	"fmt"
+	"github.com/keyval-dev/odigos/autoscaler/controllers/datacollection/custom"
 
 	"github.com/ghodss/yaml"
 	odigosv1 "github.com/keyval-dev/odigos/api/odigos/v1alpha1"
@@ -19,7 +20,7 @@ import (
 )
 
 const (
-	configKey = "collector-conf"
+	configKey = "conf"
 )
 
 func syncConfigMap(apps *odigosv1.InstrumentedApplicationList, dests *odigosv1.DestinationList,
@@ -96,6 +97,10 @@ func getDesiredConfigMap(apps *odigosv1.InstrumentedApplicationList, dests *odig
 		},
 	}
 
+	if custom.ShouldApplyCustomDataCollection(dests) {
+		custom.AddCustomConfigMap(dests, &desired)
+	}
+
 	if err := ctrl.SetControllerReference(datacollection, &desired, scheme); err != nil {
 		return nil, err
 	}
@@ -141,11 +146,11 @@ func getConfigMapData(apps *odigosv1.InstrumentedApplicationList, dests *odigosv
 	collectLogs := false
 	for _, dst := range dests.Items {
 		for _, s := range dst.Spec.Signals {
-			if s == common.LogsObservabilitySignal {
+			if s == common.LogsObservabilitySignal && !custom.DestRequiresCustom(dst.Spec.Type) {
 				collectLogs = true
 			} else if s == common.TracesObservabilitySignal {
 				collectTraces = true
-			} else if s == common.MetricsObservabilitySignal {
+			} else if s == common.MetricsObservabilitySignal && !custom.DestRequiresCustom(dst.Spec.Type) {
 				collectMetrics = true
 			}
 		}

autoscaler/controllers/datacollection/custom/custom.go

@@ -0,0 +1,53 @@
+package custom
+
+import (
+	odigosv1 "github.com/keyval-dev/odigos/api/odigos/v1alpha1"
+	"github.com/keyval-dev/odigos/common"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func ShouldApplyCustomDataCollection(dests *odigosv1.DestinationList) bool {
+	for _, dst := range dests.Items {
+		if destRequiresCustomDataCollection(dst) {
+			return true
+		}
+	}
+	return false
+}
+
+func destRequiresCustomDataCollection(dest odigosv1.Destination) bool {
+	if DestRequiresCustom(dest.Spec.Type) {
+		for _, s := range dest.Spec.Signals {
+			if s != common.TracesObservabilitySignal {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func DestRequiresCustom(destType common.DestinationType) bool {
+	switch destType {
+	case common.HoneycombDestinationType:
+		return true
+	default:
+		return false
+	}
+}
+
+func AddCustomConfigMap(dests *odigosv1.DestinationList, cm *corev1.ConfigMap) {
+	addHoneycombConfig(cm)
+}
+
+func ApplyCustomChangesToDaemonSet(ds *v1.DaemonSet, dests *odigosv1.DestinationList) {
+	secretName := ""
+	for _, dst := range dests.Items {
+		if dst.Spec.Type == common.HoneycombDestinationType {
+			secretName = dst.Spec.SecretRef.Name
+			break
+		}
+	}
+	addHoneycombToDaemonSet(ds, secretName)
+}

autoscaler/controllers/datacollection/custom/honeycomb.go

@@ -0,0 +1,108 @@
+package custom
+
+import (
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	honeycombDataCollectionImage = "honeycombio/honeycomb-kubernetes-agent:2.6.0"
+	honeycombConfigMountPath     = "/etc/honeycomb"
+	honeycombConfigKey           = "honeycomb-conf"
+)
+
+func addHoneycombConfig(cm *corev1.ConfigMap) {
+	cm.Data[honeycombConfigKey] = `    apiHost: https://api.honeycomb.io/
+    watchers:
+      - dataset: kubernetes-logs
+        labelSelector: "component=kube-apiserver,tier=control-plane"
+        namespace: kube-system
+        parser: glog
+      - dataset: kubernetes-logs
+        labelSelector: "component=kube-scheduler,tier=control-plane"
+        namespace: kube-system
+        parser: glog
+      - dataset: kubernetes-logs
+        labelSelector: "component=kube-controller-manager,tier=control-plane"
+        namespace: kube-system
+        parser: glog
+      - dataset: kubernetes-logs
+        labelSelector: "k8s-app=kube-proxy"
+        namespace: kube-system
+        parser: glog
+      - dataset: kubernetes-logs
+        labelSelector: "k8s-app=kube-dns"
+        namespace: kube-system
+        parser: glog
+    verbosity: info
+    splitLogging: false
+    metrics:
+      clusterName: k8s-cluster
+      dataset: kubernetes-metrics
+      enabled: true
+      metricGroups:
+      - node
+      - pod`
+}
+
+func addHoneycombToDaemonSet(ds *v1.DaemonSet, secretName string) {
+	ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, corev1.Container{
+		Name:  "honeycomb-collector",
+		Image: honeycombDataCollectionImage,
+		Env: []corev1.EnvVar{
+			{
+				Name: "NODE_NAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "spec.nodeName",
+					},
+				},
+			},
+			{
+				Name: "NODE_IP",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "status.hostIP",
+					},
+				},
+			},
+			{
+				Name: "HONEYCOMB_APIKEY",
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: secretName,
+						},
+						Key: "HONEYCOMB_API_KEY",
+					},
+				},
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      "varlibdockercontainers",
+				MountPath: "/var/lib/docker/containers",
+				ReadOnly:  true,
+			},
+			{
+				Name:      "varlog",
+				MountPath: "/var/log",
+				ReadOnly:  true,
+			},
+			{
+				Name:      "conf",
+				MountPath: honeycombConfigMountPath,
+				ReadOnly:  false,
+			},
+		},
+	})
+
+	for i, vol := range ds.Spec.Template.Spec.Volumes {
+		if vol.Name == "conf" {
+			ds.Spec.Template.Spec.Volumes[i].VolumeSource.ConfigMap.Items = append(ds.Spec.Template.Spec.Volumes[i].VolumeSource.ConfigMap.Items, corev1.KeyToPath{
+				Key:  honeycombConfigKey,
+				Path: "config.yaml",
+			})
+		}
+	}
+}

autoscaler/controllers/datacollection/daemonset.go

@@ -3,6 +3,7 @@ package datacollection
 import (
 	"context"
 	"fmt"
+	"github.com/keyval-dev/odigos/autoscaler/controllers/datacollection/custom"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	odigosv1 "github.com/keyval-dev/odigos/api/odigos/v1alpha1"
@@ -33,7 +34,7 @@ var (
 	}
 )
 
-func syncDaemonSet(datacollection *odigosv1.CollectorsGroup, configData string, ctx context.Context,
+func syncDaemonSet(apps *odigosv1.InstrumentedApplicationList, dests *odigosv1.DestinationList, datacollection *odigosv1.CollectorsGroup, configData string, ctx context.Context,
 	c client.Client, scheme *runtime.Scheme) (*appsv1.DaemonSet, error) {
 	logger := log.FromContext(ctx)
 	desiredDs, err := getDesiredDaemonSet(datacollection, configData, scheme)
@@ -42,6 +43,10 @@ func syncDaemonSet(datacollection *odigosv1.CollectorsGroup, configData string,
 		return nil, err
 	}
 
+	if custom.ShouldApplyCustomDataCollection(dests) {
+		custom.ApplyCustomChangesToDaemonSet(desiredDs, dests)
+	}
+
 	existing := &appsv1.DaemonSet{}
 	if err := c.Get(ctx, client.ObjectKey{Namespace: datacollection.Namespace, Name: datacollection.Name}, existing); err != nil {
 		if apierrors.IsNotFound(err) {

autoscaler/controllers/datacollection/root.go

@@ -55,7 +55,7 @@ func syncDataCollection(instApps *odigosv1.InstrumentedApplicationList, dests *o
 		return err
 	}
 
-	ds, err := syncDaemonSet(dataCollection, configData, ctx, c, scheme)
+	ds, err := syncDaemonSet(instApps, dests, dataCollection, configData, ctx, c, scheme)
 	if err != nil {
 		logger.Error(err, "failed to sync daemon set")
 		return err

cli/cmd/resources/datacollection.go

@@ -34,10 +34,14 @@ func NewDataCollectionClusterRole() *rbacv1.ClusterRole {
 			{
 				Verbs: []string{
 					"get",
+					"list",
+					"watch",
 				},
 				APIGroups: []string{""},
 				Resources: []string{
+					"pods",
 					"nodes/stats",
+					"nodes/proxy",
 				},
 			},
 		},

ui/vendors/honeycomb.tsx

@@ -11,7 +11,11 @@ export class Honeycomb implements ObservabilityVendor {
   name = "honeycomb";
   displayName = "Honeycomb";
   type = VendorType.MANAGED;
-  supportedSignals = [ObservabilitySignals.Traces];
+  supportedSignals = [
+    ObservabilitySignals.Traces,
+    ObservabilitySignals.Metrics,
+    ObservabilitySignals.Logs,
+  ];
 
   getLogo = (props: any) => {
     return <HoneycombLogo {...props} />;