Added a CS3API compliant data exporter to Mentix (#955)

docs/content/en/docs/config/http/services/mentix/_index.md

@@ -39,6 +39,7 @@ Supported values are:
 
 - **webapi**
 Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future.
+- **cs3api** Similar to the WebAPI exporter, the `cs3api` exporter exposes its data via an HTTP endpoint. Data can be retrieved at the configured relative endpoint (see [here](cs3api)). The data is compliant with the CS3API `ProviderInfo` structure.  
 - **prom_filesd**
 [Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)):
   ``` scrape_configs:

docs/content/en/docs/config/http/services/mentix/cs3api/_index.md

@@ -0,0 +1,19 @@
+---
+title: "cs3api"
+linkTitle: "cs3api"
+weight: 10
+description: >
+    Configuration for the CS3API of the Mentix service
+---
+
+{{% pageinfo %}}
+The CS3API exporter exposes Mentix data in a format that is compliant with the CS3API `ProviderInfo` structure via an HTTP endpoint.
+{{% /pageinfo %}}
+
+{{% dir name="endpoint" type="string" default="/" %}}
+The endpoint where the mesh data can be queried.
+{{< highlight toml >}}
+[http.services.mentix.cs3api]
+endpoint = "/data"
+{{< /highlight >}}
+{{% /dir %}}

docs/content/en/docs/config/http/services/mentix/webapi/_index.md

@@ -7,13 +7,13 @@ description: >
 ---
 
 {{% pageinfo %}}
-The WebAPI exporter supports multiple endpoints for exporting data. As there currently is only one such endpoint, the WebAPI settings should not be modified.
+The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint.
 {{% /pageinfo %}}
 
 {{% dir name="endpoint" type="string" default="/" %}}
 The endpoint where the mesh data can be queried.
 {{< highlight toml >}}
 [http.services.mentix.webapi]
-endpoint = "data"
+endpoint = "/data"
 {{< /highlight >}}
 {{% /dir %}}

examples/mentix/mentix.toml

@@ -7,9 +7,9 @@ enabled_services = ["mentix"]
 
 [http.services.mentix]
 connector = "gocdb"
-exporters = ["webapi"]
+exporters = ["webapi", "cs3api"]
 # Enable the Prometheus File Service Discovery:
-# exporters = ["webapi", "prom_filesd"]
+# exporters = ["webapi", "cs3api", "prom_filesd"]
 update_interval = "15m"
 
 [http.services.mentix.gocdb]
@@ -18,6 +18,9 @@ address = "http://sciencemesh-test.uni-muenster.de"
 [http.services.mentix.webapi]
 endpoint = "/"
 
+[http.services.mentix.cs3api]
+endpoint = "/cs3"
+
 # Configure the Prometheus File Service Discovery:
 # [http.services.mentix.prom_filesd]
 # Prometheus must be configured to read the following file:

internal/http/services/mentix/mentix.go

@@ -116,6 +116,10 @@ func applyDefaultConfig(conf *config.Configuration) {
 	if conf.WebAPI.Endpoint == "" {
 		conf.WebAPI.Endpoint = "/"
 	}
+
+	if conf.CS3API.Endpoint == "" {
+		conf.CS3API.Endpoint = "/cs3"
+	}
 }
 
 // New returns a new Mentix service.

pkg/mentix/config/config.go

@@ -35,6 +35,10 @@ type Configuration struct {
 		Endpoint string `mapstructure:"endpoint"`
 	} `yaml:"webapi"`
 
+	CS3API struct {
+		Endpoint string `mapstructure:"endpoint"`
+	} `yaml:"cs3api"`
+
 	PrometheusFileSD struct {
 		OutputFile string `mapstructure:"output_file"`
 	} `mapstructure:"prom_filesd"`

pkg/mentix/config/ids.go

@@ -19,10 +19,15 @@
 package config
 
 const (
+	// ConnectorIDGOCDB is the connector identifier for GOCDB.
 	ConnectorIDGOCDB = "gocdb"
 )
 
 const (
-	ExporterIDWebAPI           = "webapi"
+	// ExporterIDWebAPI is the identifier for the WebAPI exporter.
+	ExporterIDWebAPI = "webapi"
+	// ExporterIDCS3API is the identifier for the CS3API exporter.
+	ExporterIDCS3API = "cs3api"
+	// ExporterIDPrometheusFileSD is the identifier for the Prometheus File SD exporter.
 	ExporterIDPrometheusFileSD = "prom_filesd"
 )

pkg/mentix/connectors/connector.go

@@ -49,6 +49,7 @@ type BaseConnector struct {
 	log  *zerolog.Logger
 }
 
+// Activate activates the connector.
 func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
 	if conf == nil {
 		return fmt.Errorf("no configuration provided")
@@ -63,6 +64,7 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo
 	return nil
 }
 
+// FindConnector searches for the given connector ID in all globally registered connectors.
 func FindConnector(connectorID string) (Connector, error) {
 	for id, connector := range registeredConnectors {
 		if strings.EqualFold(id, connectorID) {

pkg/mentix/connectors/gocdb.go

@@ -39,6 +39,7 @@ type GOCDBConnector struct {
 	gocdbAddress string
 }
 
+// Activate activates the connector.
 func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
 	if err := connector.BaseConnector.Activate(conf, log); err != nil {
 		return err
@@ -53,6 +54,7 @@ func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerol
 	return nil
 }
 
+// RetrieveMeshData fetches new mesh data.
 func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) {
 	meshData := new(meshdata.MeshData)
 
@@ -188,7 +190,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site
 
 		// Add the service to the site
 		site.Services = append(site.Services, &meshdata.Service{
-			ServiceEndpoint: meshdata.ServiceEndpoint{
+			ServiceEndpoint: &meshdata.ServiceEndpoint{
 				Type:        connector.findServiceType(meshData, service.Type),
 				Name:        fmt.Sprintf("%v - %v", service.Host, service.Type),
 				URL:         getServiceURLString(service, nil, host),
@@ -252,6 +254,7 @@ func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint
 	return svcURL, nil
 }
 
+// GetName returns the display name of the connector.
 func (connector *GOCDBConnector) GetName() string {
 	return "GOCDB"
 }

pkg/mentix/exporters/cs3api.go

@@ -0,0 +1,73 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package exporters
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/rs/zerolog"
+
+	"github.com/cs3org/reva/pkg/mentix/config"
+	"github.com/cs3org/reva/pkg/mentix/exporters/cs3api"
+)
+
+// CS3APIExporter implements the CS3API exporter.
+type CS3APIExporter struct {
+	BaseRequestExporter
+}
+
+// Activate activates the exporter.
+func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
+	if err := exporter.BaseExporter.Activate(conf, log); err != nil {
+		return err
+	}
+
+	// Store CS3API specific settings
+	exporter.endpoint = conf.CS3API.Endpoint
+
+	return nil
+}
+
+// HandleRequest handles the actual HTTP request.
+func (exporter *CS3APIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error {
+	// Data is read, so acquire a read lock
+	exporter.locker.RLock()
+	defer exporter.locker.RUnlock()
+
+	data, err := cs3api.HandleQuery(exporter.meshData, req.URL.Query())
+	if err == nil {
+		if _, err := resp.Write(data); err != nil {
+			return fmt.Errorf("error writing the API request response: %v", err)
+		}
+	} else {
+		return fmt.Errorf("error while serving API request: %v", err)
+	}
+
+	return nil
+}
+
+// GetName returns the display name of the exporter.
+func (exporter *CS3APIExporter) GetName() string {
+	return "CS3API"
+}
+
+func init() {
+	registerExporter(config.ExporterIDCS3API, &CS3APIExporter{})
+}

pkg/mentix/exporters/cs3api/query.go

@@ -0,0 +1,113 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package cs3api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+
+	ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1"
+
+	"github.com/cs3org/reva/pkg/mentix/meshdata"
+)
+
+const (
+	queryMethodDefault = ""
+)
+
+// HandleQuery handles an HTTP request based on the provided 'method' parameter.
+func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) {
+	method := params.Get("method")
+	switch strings.ToLower(method) {
+	case queryMethodDefault:
+		return handleDefaultQuery(meshData, params)
+
+	default:
+		return []byte{}, fmt.Errorf("unknown API method '%v'", method)
+	}
+}
+
+func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) {
+	// Convert the mesh data
+	ocmData, err := convertMeshDataToOCMData(meshData)
+	if err != nil {
+		return []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err)
+	}
+
+	// Marshal the OCM data as JSON
+	data, err := json.MarshalIndent(ocmData, "", "\t")
+	if err != nil {
+		return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err)
+	}
+
+	return data, nil
+}
+
+func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) {
+	// Convert the mesh data into the corresponding OCM data structures
+	providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Sites))
+	for _, site := range meshData.Sites {
+		// Gather all services from the site
+		services := make([]*ocmprovider.Service, 0, len(site.Services))
+		for _, service := range site.Services {
+			// Gather all additional endpoints of the service
+			addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints))
+			for _, endpoint := range service.AdditionalEndpoints {
+				addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint))
+			}
+
+			services = append(services, &ocmprovider.Service{
+				Host:                service.Host,
+				Endpoint:            convertServiceEndpointToOCMData(service.ServiceEndpoint),
+				AdditionalEndpoints: addEndpoints,
+				ApiVersion:          meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, ""),
+			})
+		}
+
+		// Copy the site info into a ProviderInfo
+		providers = append(providers, &ocmprovider.ProviderInfo{
+			Name:         site.Name,
+			FullName:     site.FullName,
+			Description:  site.Description,
+			Organization: site.Organization,
+			Domain:       site.Domain,
+			Homepage:     site.Homepage,
+			Email:        site.Email,
+			Services:     services,
+			Properties:   site.Properties,
+		})
+	}
+
+	return providers, nil
+}
+
+func convertServiceEndpointToOCMData(endpoint *meshdata.ServiceEndpoint) *ocmprovider.ServiceEndpoint {
+	return &ocmprovider.ServiceEndpoint{
+		Type: &ocmprovider.ServiceType{
+			Name:        endpoint.Type.Name,
+			Description: endpoint.Type.Description,
+		},
+		Name:        endpoint.Name,
+		Path:        endpoint.URL,
+		IsMonitored: endpoint.IsMonitored,
+		Properties:  endpoint.Properties,
+	}
+}

pkg/mentix/exporters/exporter.go

@@ -57,6 +57,7 @@ type BaseExporter struct {
 	locker   sync.RWMutex
 }
 
+// Activate activates the exporter.
 func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
 	if conf == nil {
 		return fmt.Errorf("no configuration provided")
@@ -71,14 +72,17 @@ func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog.
 	return nil
 }
 
+// Start starts the exporter; only exporters which perform periodical background tasks should do something here.
 func (exporter *BaseExporter) Start() error {
 	return nil
 }
 
+// Stop stops any running background activities of the exporter.
 func (exporter *BaseExporter) Stop() {
 
 }
 
+// UpdateMeshData is called whenever the mesh data has changed to reflect these changes.
 func (exporter *BaseExporter) UpdateMeshData(meshData *meshdata.MeshData) error {
 	// Update the stored mesh data
 	if err := exporter.storeMeshData(meshData); err != nil {