xcloud metrics gathering (#1403)

changelog/unreleased/xcloud-metrics.md

@@ -0,0 +1,7 @@
+Enhancement: support remote cloud gathering metrics
+
+The current metrics package can only gather metrics either from
+json files. With this feature, the metrics can be gathered polling
+the http endpoints exposed by the owncloud/nextcloud sciencemesh apps.
+
+https://github.com/cs3org/reva/pull/1403

examples/metrics/xcloud.toml

@@ -0,0 +1,11 @@
+[http]
+address = "0.0.0.0:5550"
+
+[http.services.metrics]
+metrics_data_driver_type = "xcloud"
+metrics_record_interval = 5000
+xcloud_instance = "http://localhost"
+xcloud_interval = 5
+xcloud_catalog = 'https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register'
+
+[http.services.prometheus]

pkg/metrics/config/config.go

@@ -23,6 +23,9 @@ type Config struct {
 	MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"`
 	MetricsDataLocation   string `mapstructure:"metrics_data_location"`
 	MetricsRecordInterval int    `mapstructure:"metrics_record_interval"`
+	XcloudInstance        string `mapstructure:"xcloud_instance"`
+	XcloudCatalog         string `mapstructure:"xcloud_catalog"`
+	XcloudPullInterval    int    `mapstructure:"xcloud_pull_interval"`
 }
 
 // Init sets sane defaults
@@ -33,6 +36,7 @@ func (c *Config) Init() {
 			c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json"
 		}
 	}
+
 	if c.MetricsRecordInterval == 0 {
 		c.MetricsRecordInterval = 5000
 	}

pkg/metrics/driver/loader/loader.go

@@ -22,5 +22,6 @@ import (
 	// Load metrics drivers.
 	_ "github.com/cs3org/reva/pkg/metrics/driver/dummy"
 	_ "github.com/cs3org/reva/pkg/metrics/driver/json"
+	_ "github.com/cs3org/reva/pkg/metrics/driver/xcloud"
 	// Add your own here
 )

pkg/metrics/driver/xcloud/xcloud.go

@@ -0,0 +1,269 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/cs3org/reva/pkg/metrics/driver/registry"
+
+	"github.com/cs3org/reva/pkg/logger"
+	"github.com/cs3org/reva/pkg/metrics/config"
+	"github.com/rs/zerolog"
+)
+
+var log zerolog.Logger
+
+func init() {
+	log = logger.New().With().Int("pid", os.Getpid()).Logger()
+	driver := &CloudDriver{CloudData: &CloudData{}}
+	registry.Register(driverName(), driver)
+
+}
+
+func driverName() string {
+	return "xcloud"
+}
+
+// CloudDriver is the driver to use for Sciencemesh apps
+type CloudDriver struct {
+	instance     string
+	catalog      string
+	pullInterval int
+	CloudData    *CloudData
+	sync.Mutex
+	client *http.Client
+}
+
+func (d *CloudDriver) refresh() error {
+
+	// endpoint example: https://mybox.com or https://mybox.com/owncloud
+	endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance)
+
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error creating request to %s", d.instance)
+		return err
+	}
+
+	resp, err := d.client.Do(req)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", d.instance, resp.StatusCode)
+		log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+		return err
+	}
+	defer resp.Body.Close()
+
+	// read response body
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", d.instance)
+		return err
+	}
+
+	cd := &CloudData{}
+	if err := json.Unmarshal(data, cd); err != nil {
+		log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data))
+		return err
+	}
+
+	d.Lock()
+	defer d.Unlock()
+	d.CloudData = cd
+	log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd)
+
+	mc := &MentixCatalog{
+		Name:        cd.Settings.Sitename,
+		FullName:    cd.Settings.Sitename,
+		Homepage:    cd.Settings.Hostname,
+		Description: "ScienceMesh App from " + cd.Settings.Sitename,
+		CountryCode: cd.Settings.Country,
+		Services: []*MentixService{
+			&MentixService{
+				Host:        cd.Settings.Hostname,
+				IsMonitored: true,
+				Name:        cd.Settings.Hostname + " - REVAD",
+				URL:         cd.Settings.Siteurl,
+				Properties: &MentixServiceProperties{
+					MetricsPath: "/index.php/apps/sciencemesh/metrics",
+				},
+				Type: &MentixServiceType{
+					Name: "REVAD",
+				},
+			},
+		},
+	}
+
+	j, err := json.Marshal(mc)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error marhsaling mentix calalog info")
+		return err
+	}
+
+	log.Info().Msgf("xcloud: info to send to register: %s", string(j))
+
+	// send to register if catalog is set
+	req, err = http.NewRequest("POST", d.catalog, bytes.NewBuffer(j))
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error creating POST request to: %s", d.catalog)
+		return err
+	}
+
+	resp, err = d.client.Do(req)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j))
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, _ := ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("xcloud: error registering site: status code(%d) body(%s)", resp.StatusCode, string(body))
+		log.Err(err).Msg("xcloud: error registering site")
+		return err
+	}
+
+	log.Info().Msgf("xcloud: site registered: %s", string(body))
+	return nil
+}
+
+// Configure configures this driver
+func (d *CloudDriver) Configure(c *config.Config) error {
+	if c.XcloudInstance == "" {
+		err := errors.New("xcloud: missing xcloud_instance config parameter")
+		return err
+	}
+
+	if c.XcloudPullInterval == 0 {
+		c.XcloudPullInterval = 10 // seconds
+	}
+
+	d.instance = c.XcloudInstance
+	d.pullInterval = c.XcloudPullInterval
+	d.catalog = c.XcloudCatalog
+
+	// TODO(labkode): make it configurable once site adopted are prod-ready
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+
+	d.client = client
+
+	ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second)
+	quit := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				err := d.refresh()
+				if err != nil {
+					log.Err(err).Msgf("xcloud: error from refresh goroutine")
+				}
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
+// GetNumUsers returns the number of site users
+func (d *CloudDriver) GetNumUsers() int64 {
+	return d.CloudData.Metrics.TotalUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *CloudDriver) GetNumGroups() int64 {
+	return d.CloudData.Metrics.TotalGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *CloudDriver) GetAmountStorage() int64 {
+	return d.CloudData.Metrics.TotalStorage
+}
+
+// CloudData represents the information obtained from the sciencemesh app
+type CloudData struct {
+	Metrics  CloudDataMetrics  `json:"metrics"`
+	Settings CloudDataSettings `json:"settings"`
+}
+
+// CloudDataMetrics reprents the metrics gathered from the sciencemesh app
+type CloudDataMetrics struct {
+	TotalUsers   int64 `json:"numusers"`
+	TotalGroups  int64 `json:"numgroups"`
+	TotalStorage int64 `json:"numstorage"`
+}
+
+// CloudDataSettings represents the metrics gathered
+type CloudDataSettings struct {
+	IOPUrl   string `json:"iopurl"`
+	Sitename string `json:"sitename"`
+	Siteurl  string `json:"siteurl"`
+	Hostname string `json:"hostname"`
+	Country  string `json:"country"`
+}
+
+// MentixCatalog represents the information needed to register a site into the mesh
+type MentixCatalog struct {
+	CountryCode string           `json:"CountryCode"`
+	Description string           `json:"Description"`
+	FullName    string           `json:"FullName"`
+	Homepage    string           `json:"Homepage"`
+	Name        string           `json:"Name"`
+	Services    []*MentixService `json:"Services"`
+}
+
+// MentixService represents the service running in a site
+type MentixService struct {
+	Host        string                   `json:"Host"`
+	IsMonitored bool                     `json:"IsMonitored"`
+	Name        string                   `json:"Name"`
+	Properties  *MentixServiceProperties `json:"Properties"`
+	Type        *MentixServiceType       `json:"Type"`
+	URL         string                   `json:"URL"`
+}
+
+// MentixServiceProperties represents the properties to expose the metrics endpoint
+type MentixServiceProperties struct {
+	MetricsPath string `json:"METRICS_PATH"`
+}
+
+// MentixServiceType represents the type of service running
+type MentixServiceType struct {
+	Name string `json:"Name"`
+}