diff --git a/frontend/destinations/data/chronosphere.yaml b/frontend/destinations/data/chronosphere.yaml index f8a454ae6..c0710fe01 100644 --- a/frontend/destinations/data/chronosphere.yaml +++ b/frontend/destinations/data/chronosphere.yaml @@ -14,9 +14,9 @@ spec: logs: supported: false fields: - - name: collector_endpoint + - name: CHRONOSPHERE_COLLECTOR displayName: Collector Endpoint videoUrl: https://www.youtube.com/watch?v=9QZxw-mtZmU componentType: input componentProps: - type: text \ No newline at end of file + type: text diff --git a/frontend/destinations/data/datadog.yaml b/frontend/destinations/data/datadog.yaml index 7dcc86e66..ebcce3943 100644 --- a/frontend/destinations/data/datadog.yaml +++ b/frontend/destinations/data/datadog.yaml @@ -14,13 +14,14 @@ spec: logs: supported: true fields: - - name: api_key + - name: DATADOG_API_KEY displayName: API Key videoUrl: https://www.youtube.com/watch?v=9QZxw-mtZmU componentType: input componentProps: type: password - - name: site + secret: true + - name: DATADOG_SITE displayName: Site videoUrl: https://www.youtube.com/watch?v=9QZxw-mtZmU componentType: dropdown diff --git a/frontend/destinations/data/jaeger.yaml b/frontend/destinations/data/jaeger.yaml index a3a5d7342..685040ae9 100644 --- a/frontend/destinations/data/jaeger.yaml +++ b/frontend/destinations/data/jaeger.yaml @@ -14,9 +14,9 @@ spec: logs: supported: false fields: - - name: endpoint + - name: JAEGER_URL displayName: Endpoint videoUrl: https://www.youtube.com/watch?v=9QZxw-mtZmU componentType: input componentProps: - type: text \ No newline at end of file + type: text diff --git a/frontend/destinations/model.go b/frontend/destinations/model.go index 82495bd78..d35a66df3 100644 --- a/frontend/destinations/model.go +++ b/frontend/destinations/model.go @@ -1,5 +1,7 @@ package destinations +import "github.com/keyval-dev/odigos/common" + type Destination struct { ApiVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` @@ -8,9 +10,9 @@ type Destination struct { } type Metadata struct { - Type string `yaml:"type"` - DisplayName string `yaml:"displayName"` - Category string `yaml:"category"` + Type common.DestinationType `yaml:"type"` + DisplayName string `yaml:"displayName"` + Category string `yaml:"category"` } type Spec struct { @@ -35,4 +37,5 @@ type Field struct { VideoURL string `yaml:"videoUrl"` ComponentType string `yaml:"componentType"` ComponentProps map[string]interface{} `yaml:"componentProps"` + Secret bool `yaml:"secret"` } diff --git a/frontend/endpoints/common.go b/frontend/endpoints/common.go index 0f9a50ce9..4d3256ac8 100644 --- a/frontend/endpoints/common.go +++ b/frontend/endpoints/common.go @@ -11,3 +11,13 @@ func returnError(c *gin.Context, err error) { "message": err.Error(), }) } + +func returnErrors(c *gin.Context, errors []error) { + errorsText := make([]string, len(errors)) + for i, err := range errors { + errorsText[i] = err.Error() + } + c.JSON(http.StatusInternalServerError, gin.H{ + "messages": errorsText, + }) +} diff --git a/frontend/endpoints/destinations.go b/frontend/endpoints/destinations.go index 8eabfb118..4e538b865 100644 --- a/frontend/endpoints/destinations.go +++ b/frontend/endpoints/destinations.go @@ -1,52 +1,72 @@ package endpoints import ( + "fmt" + "github.com/gin-gonic/gin" + "github.com/keyval-dev/odigos/api/odigos/v1alpha1" + "github.com/keyval-dev/odigos/common" "github.com/keyval-dev/odigos/frontend/destinations" + "github.com/keyval-dev/odigos/frontend/kube" + k8s "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type GetDestinationsResponse struct { +type GetDestinationTypesResponse struct { Categories []DestinationsCategory `json:"categories"` } type DestinationsCategory struct { - Name string `json:"name"` - Items []DestinationsCategoryItem `json:"items"` + Name string `json:"name"` + Items []DestinationTypesCategoryItem `json:"items"` } -type DestinationsCategoryItem struct { - Type string `json:"type"` - DisplayName string `json:"display_name"` - ImageUrl string `json:"image_url"` - SupportedSignals SupportedSignals `json:"supported_signals"` +type DestinationTypesCategoryItem struct { + Type common.DestinationType `json:"type"` + DisplayName string `json:"display_name"` + ImageUrl string `json:"image_url"` + SupportedSignals SupportedSignals `json:"supported_signals"` } type SupportedSignals struct { - Traces ObservabilitySignal `json:"traces"` - Metrics ObservabilitySignal `json:"metrics"` - Logs ObservabilitySignal `json:"logs"` + Traces ObservabilitySignalSupport `json:"traces"` + Metrics ObservabilitySignalSupport `json:"metrics"` + Logs ObservabilitySignalSupport `json:"logs"` } -type ObservabilitySignal struct { +type ObservabilitySignalSupport struct { Supported bool `json:"supported"` } -func GetDestinations(c *gin.Context) { - var resp GetDestinationsResponse - itemsByCategory := make(map[string][]DestinationsCategoryItem) +type ExportedSignals struct { + Traces bool `json:"traces"` + Metrics bool `json:"metrics"` + Logs bool `json:"logs"` +} + +type Destination struct { + Name string `json:"name"` + Type common.DestinationType `json:"type"` + ExportedSignals ExportedSignals `json:"signals"` + Data map[string]string `json:"data"` +} + +func GetDestinationTypes(c *gin.Context) { + var resp GetDestinationTypesResponse + itemsByCategory := make(map[string][]DestinationTypesCategoryItem) for _, dest := range destinations.Get() { - item := DestinationsCategoryItem{ + item := DestinationTypesCategoryItem{ Type: dest.Metadata.Type, DisplayName: dest.Metadata.DisplayName, ImageUrl: GetImageURL(dest.Spec.Image), SupportedSignals: SupportedSignals{ - Traces: ObservabilitySignal{ + Traces: ObservabilitySignalSupport{ Supported: dest.Spec.Signals.Traces.Supported, }, - Metrics: ObservabilitySignal{ + Metrics: ObservabilitySignalSupport{ Supported: dest.Spec.Signals.Metrics.Supported, }, - Logs: ObservabilitySignal{ + Logs: ObservabilitySignalSupport{ Supported: dest.Spec.Signals.Logs.Supported, }, }, @@ -77,27 +97,346 @@ type Field struct { VideoUrl string `json:"video_url"` } -func GetDestinationDetails(c *gin.Context) { - destType := c.Param("type") +func GetDestinationTypeDetails(c *gin.Context) { + destType := common.DestinationType(c.Param("type")) + destTypeConfig, err := getDestinationTypeConfig(destType) + if err != nil { + c.JSON(404, gin.H{ + "error": fmt.Sprintf("destination type %s not found", destType), + }) + return + } + + var resp GetDestinationDetailsResponse + for _, field := range destTypeConfig.Spec.Fields { + resp.Fields = append(resp.Fields, Field{ + Name: field.Name, + DisplayName: field.DisplayName, + ComponentType: field.ComponentType, + ComponentProperties: field.ComponentProps, + VideoUrl: field.VideoURL, + }) + } + + c.JSON(200, resp) + return +} + +func GetDestinations(c *gin.Context, odigosns string) { + dests, err := kube.DefaultClient.OdigosClient.Destinations(odigosns).List(c, metav1.ListOptions{}) + if err != nil { + returnError(c, err) + return + } + + var resp []Destination + for _, dest := range dests.Items { + secretFields, err := getDestinationSecretFields(c, odigosns, &dest) + if err != nil { + returnError(c, err) + return + } + endpointDest := k8sDestinationToEndpointFormat(dest, secretFields) + resp = append(resp, endpointDest) + } + + c.JSON(200, resp) +} + +func GetDestinationByName(c *gin.Context, odigosns string) { + destName := c.Param("name") + destination, err := kube.DefaultClient.OdigosClient.Destinations(odigosns).Get(c, destName, metav1.GetOptions{}) + if err != nil { + returnError(c, err) + return + } + + secretFields, err := getDestinationSecretFields(c, odigosns, destination) + if err != nil { + returnError(c, err) + return + } + resp := k8sDestinationToEndpointFormat(*destination, secretFields) + c.JSON(200, resp) +} + +func CreateNewDestination(c *gin.Context, odigosns string) { + + request := Destination{} + if err := c.ShouldBindJSON(&request); err != nil { + returnError(c, err) + return + } + + destType := request.Type + destName := request.Name + + destTypeConfig, err := getDestinationTypeConfig(destType) + if err != nil { + returnError(c, err) + return + } + + errors := verifyDestinationDataScheme(destType, destTypeConfig, request.Data) + if len(errors) > 0 { + returnErrors(c, errors) + return + } + + dataField, secretFields := transformFieldsToDataAndSecrets(destTypeConfig, request.Data) + + destSpec := v1alpha1.DestinationSpec{ + Type: destType, + Data: dataField, + Signals: exportedSignalsObjectToSlice(request.ExportedSignals), + } + + if len(secretFields) > 0 { + destSpec.SecretRef = &k8s.LocalObjectReference{ + Name: destName, + } + secret := k8s.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: destName, + }, + StringData: secretFields, + } + _, err := kube.DefaultClient.CoreV1().Secrets(odigosns).Create(c, &secret, metav1.CreateOptions{}) + if err != nil { + returnError(c, err) + return + } + } + + k8sDestination := v1alpha1.Destination{ + ObjectMeta: metav1.ObjectMeta{ + Name: destName, + }, + Spec: destSpec, + } + dest, err := kube.DefaultClient.OdigosClient.Destinations(odigosns).Create(c, &k8sDestination, metav1.CreateOptions{}) + if err != nil { + returnError(c, err) + return + } + + resp := k8sDestinationToEndpointFormat(*dest, secretFields) + c.JSON(201, resp) +} + +func UpdateExistingDestination(c *gin.Context, odigosns string) { + request := Destination{} + if err := c.ShouldBindJSON(&request); err != nil { + returnError(c, err) + return + } + + destType := request.Type + destName := request.Name + + destTypeConfig, err := getDestinationTypeConfig(destType) + if err != nil { + returnError(c, err) + return + } + + errors := verifyDestinationDataScheme(destType, destTypeConfig, request.Data) + if len(errors) > 0 { + returnErrors(c, errors) + return + } + + dataFields, secretFields := transformFieldsToDataAndSecrets(destTypeConfig, request.Data) + + // update destination + dest, err := kube.DefaultClient.OdigosClient.Destinations(odigosns).Get(c, destName, metav1.GetOptions{}) + if err != nil { + returnError(c, err) + return + } + + secretRef := dest.Spec.SecretRef + if secretRef != nil { + secret, err := kube.DefaultClient.CoreV1().Secrets(odigosns).Get(c, secretRef.Name, metav1.GetOptions{}) + if err != nil { + returnError(c, err) + return + } + secret.StringData = secretFields + _, err = kube.DefaultClient.CoreV1().Secrets(odigosns).Update(c, secret, metav1.UpdateOptions{}) + if err != nil { + returnError(c, err) + return + } + } + + dest.Spec.Type = request.Type + dest.Spec.Data = dataFields + dest.Spec.Signals = exportedSignalsObjectToSlice(request.ExportedSignals) + + updatedDest, err := kube.DefaultClient.OdigosClient.Destinations(odigosns).Update(c, dest, metav1.UpdateOptions{}) + if err != nil { + returnError(c, err) + return + } + + resp := k8sDestinationToEndpointFormat(*updatedDest, secretFields) + c.JSON(201, resp) +} + +func DeleteDestination(c *gin.Context, odigosns string) { + destName := c.Param("name") + errDest := kube.DefaultClient.OdigosClient.Destinations(odigosns).Delete(c, destName, metav1.DeleteOptions{}) + errSecret := kube.DefaultClient.CoreV1().Secrets(odigosns).Delete(c, destName, metav1.DeleteOptions{}) + + if errDest != nil { + returnError(c, errDest) + return + } + + if errSecret != nil { + returnError(c, errDest) + return + } + + c.Status(204) +} + +func k8sDestinationToEndpointFormat(k8sDest v1alpha1.Destination, secretFields map[string]string) Destination { + destType := k8sDest.Spec.Type + destName := k8sDest.Name + mergedFields := mergeDataAndSecrets(k8sDest.Spec.Data, secretFields) + + return Destination{ + Name: destName, + Type: destType, + ExportedSignals: ExportedSignals{ + Traces: isSignalExported(k8sDest, common.TracesObservabilitySignal), + Metrics: isSignalExported(k8sDest, common.MetricsObservabilitySignal), + Logs: isSignalExported(k8sDest, common.LogsObservabilitySignal), + }, + Data: mergedFields, + } +} + +func mergeDataAndSecrets(data map[string]string, secrets map[string]string) map[string]string { + merged := map[string]string{} + + for k, v := range data { + merged[k] = v + } + + for k, v := range secrets { + merged[k] = v + } + + return merged +} + +func isSignalExported(dest v1alpha1.Destination, signal common.ObservabilitySignal) bool { + for _, s := range dest.Spec.Signals { + if s == signal { + return true + } + } + + return false +} + +func exportedSignalsObjectToSlice(signals ExportedSignals) []common.ObservabilitySignal { + var resp []common.ObservabilitySignal + if signals.Traces { + resp = append(resp, common.TracesObservabilitySignal) + } + if signals.Metrics { + resp = append(resp, common.MetricsObservabilitySignal) + } + if signals.Logs { + resp = append(resp, common.LogsObservabilitySignal) + } + + return resp +} + +func verifyDestinationDataScheme(destType common.DestinationType, destTypeConfig *destinations.Destination, data map[string]string) []error { + + errors := []error{} + + // verify all fields in config are present in data (assuming here all fields are required) + for _, field := range destTypeConfig.Spec.Fields { + fieldValue, found := data[field.Name] + if !found || fieldValue == "" { + errors = append(errors, fmt.Errorf("field %s is required", field.Name)) + } + } + + // verify data fields are found in config + for dataField := range data { + found := false + // iterating all fields in config every time, assuming it's a small list + for _, field := range destTypeConfig.Spec.Fields { + if dataField == field.Name { + found = true + break + } + } + if !found { + errors = append(errors, fmt.Errorf("field %s is not found in config for destination type '%s'", dataField, destType)) + } + } + + return errors +} + +func getDestinationTypeConfig(destType common.DestinationType) (*destinations.Destination, error) { for _, dest := range destinations.Get() { if dest.Metadata.Type == destType { - var resp GetDestinationDetailsResponse - for _, field := range dest.Spec.Fields { - resp.Fields = append(resp.Fields, Field{ - Name: field.Name, - DisplayName: field.DisplayName, - ComponentType: field.ComponentType, - ComponentProperties: field.ComponentProps, - VideoUrl: field.VideoURL, - }) - } + return &dest, nil + } + } - c.JSON(200, resp) - return + return nil, fmt.Errorf("destination type %s not found", destType) +} + +func transformFieldsToDataAndSecrets(destTypeConfig *destinations.Destination, fields map[string]string) (map[string]string, map[string]string) { + + dataFields := map[string]string{} + secretFields := map[string]string{} + + for fieldName, fieldValue := range fields { + // for each field in the data, find it's config + // assuming the list is small so it's ok to iterate it + for _, fieldConfig := range destTypeConfig.Spec.Fields { + if fieldName == fieldConfig.Name { + if fieldConfig.Secret { + secretFields[fieldName] = fieldValue + } else { + dataFields[fieldName] = fieldValue + } + } } } - c.JSON(404, gin.H{ - "error": "destination not found", - }) + return dataFields, secretFields +} + +func getDestinationSecretFields(c *gin.Context, odigosns string, dest *v1alpha1.Destination) (map[string]string, error) { + + secretFields := map[string]string{} + secretRef := dest.Spec.SecretRef + + if secretRef == nil { + return secretFields, nil + } + + secret, err := kube.DefaultClient.CoreV1().Secrets(odigosns).Get(c, secretRef.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + for k, v := range secret.Data { + secretFields[k] = string(v) + } + + return secretFields, nil } diff --git a/frontend/main.go b/frontend/main.go index 7b90cafcf..73db53acd 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -9,6 +9,7 @@ import ( "net/http" "path/filepath" + "github.com/keyval-dev/odigos/common/consts" "github.com/keyval-dev/odigos/frontend/destinations" "github.com/gin-contrib/cors" @@ -31,6 +32,7 @@ type Flags struct { Port int Debug bool KubeConfig string + Namespace string } //go:embed all:webapp/out/* @@ -47,6 +49,7 @@ func parseFlags() Flags { flag.IntVar(&flags.Port, "port", defaultPort, "Port to listen on") flag.BoolVar(&flags.Debug, "debug", false, "Enable debug mode") flag.StringVar(&flags.KubeConfig, "kubeconfig", defaultKubeConfig, "Path to kubeconfig file") + flag.StringVar(&flags.Namespace, "namespace", consts.DefaultNamespace, "Kubernetes namespace where odigos is installed") flag.Parse() return flags } @@ -90,8 +93,13 @@ func startHTTPServer(flags *Flags) (*gin.Engine, error) { apis.POST("/namespaces", endpoints.PersistNamespaces) apis.GET("/applications/:namespace", endpoints.GetApplicationsInNamespace) apis.GET("/config", endpoints.GetConfig) - apis.GET("/destinations", endpoints.GetDestinations) - apis.GET("/destinations/:type", endpoints.GetDestinationDetails) + apis.GET("/destination-types", endpoints.GetDestinationTypes) + apis.GET("/destination-types/:type", endpoints.GetDestinationTypeDetails) + apis.GET("/destinations", func(c *gin.Context) { endpoints.GetDestinations(c, flags.Namespace) }) + apis.GET("/destinations/:name", func (c *gin.Context) { endpoints.GetDestinationByName(c, flags.Namespace) }) + apis.POST("/destinations", func(c *gin.Context) { endpoints.CreateNewDestination(c, flags.Namespace) }) + apis.PUT("/destinations", func(c *gin.Context) { endpoints.UpdateExistingDestination(c, flags.Namespace) }) + apis.DELETE("/destinations/:name", func(c *gin.Context) { endpoints.DeleteDestination(c, flags.Namespace) }) } return r, nil diff --git a/frontend/webapp/utils/constants/urls.tsx b/frontend/webapp/utils/constants/urls.tsx index 19683f475..5b9e65078 100644 --- a/frontend/webapp/utils/constants/urls.tsx +++ b/frontend/webapp/utils/constants/urls.tsx @@ -7,7 +7,7 @@ const API = { CONFIG: `${BASE_URL}/config`, NAMESPACES: `${BASE_URL}/namespaces`, APPLICATIONS: `${BASE_URL}/applications`, - DESTINATION: `${BASE_URL}/destinations`, + DESTINATION: `${BASE_URL}/destination-types`, }; const QUERIES = {