From c8e1c1702b6922758c3b40438c1fe4a057e3c7d7 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Tue, 27 Oct 2020 16:21:42 +0100 Subject: [PATCH 01/35] Add site types --- pkg/mentix/connectors/gocdb.go | 1 + pkg/mentix/meshdata/site.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 46ab6c0824..38784d7ab3 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -130,6 +130,7 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) meshsite := &meshdata.Site{ + Type: meshdata.SiteTypeScienceMesh, // All sites stored in the GOCDB are part of the mesh Name: site.ShortName, FullName: site.OfficialName, Organization: organization, diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index f799030a3e..f19fd929b3 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -18,8 +18,19 @@ package meshdata +const ( + // SiteTypeScienceMesh flags a site as being part of the mesh. + SiteTypeScienceMesh = iota + // SiteTypeCommunity flags a site as being a community site. + SiteTypeCommunity +) + +type SiteType int + // Site represents a single site managed by Mentix. type Site struct { + Type SiteType `json:"-"` + Name string FullName string Organization string From 09e2c37a2ca4dfe552fbb53c8d9e41abff009924 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 10:13:32 +0100 Subject: [PATCH 02/35] Add a connector for local files --- internal/http/services/mentix/mentix.go | 4 +- pkg/mentix/config/config.go | 14 ++-- pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/file.go | 93 +++++++++++++++++++++++++ pkg/mentix/connectors/gocdb.go | 7 +- 5 files changed, 111 insertions(+), 9 deletions(-) create mode 100755 pkg/mentix/connectors/file.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 95b234c788..0ece4feda8 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -109,8 +109,8 @@ func applyDefaultConfig(conf *config.Configuration) { conf.UpdateInterval = "1h" // Update once per hour } - if conf.GOCDB.Scope == "" { - conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future + if conf.Connectors.GOCDB.Scope == "" { + conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future } if conf.WebAPI.Endpoint == "" { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 6125cebb26..065a117cd4 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -26,10 +26,16 @@ type Configuration struct { Exporters []string `mapstructure:"exporters"` UpdateInterval string `mapstructure:"update_interval"` - GOCDB struct { - Address string `mapstructure:"address"` - Scope string `mapstructure:"scope"` - } `mapstructure:"gocdb"` + Connectors struct { + GOCDB struct { + Address string `mapstructure:"address"` + Scope string `mapstructure:"scope"` + } `mapstructure:"gocdb"` + + File struct { + File string `mapstructure:"file"` + } `mapstructure:"file"` + } `mapstructure:"connectors"` WebAPI struct { Endpoint string `mapstructure:"endpoint"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index dc5abd8dc4..5432d4b6fc 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -21,6 +21,8 @@ package config const ( // ConnectorIDGOCDB is the connector identifier for GOCDB. ConnectorIDGOCDB = "gocdb" + // ConnectorIDFile is the connector identifier for local files. + ConnectorIDFile = "file" ) const ( diff --git a/pkg/mentix/connectors/file.go b/pkg/mentix/connectors/file.go new file mode 100755 index 0000000000..fefb7b1a82 --- /dev/null +++ b/pkg/mentix/connectors/file.go @@ -0,0 +1,93 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package connectors + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +// FileConnector is used to read sites from a local file. +type FileConnector struct { + BaseConnector + + filePath string +} + +// Activate activates the connector. +func (connector *FileConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := connector.BaseConnector.Activate(conf, log); err != nil { + return err + } + + // Check and store GOCDB specific settings + connector.filePath = conf.Connectors.File.File + if len(connector.filePath) == 0 { + return fmt.Errorf("no file configured") + } + + return nil +} + +// RetrieveMeshData fetches new mesh data. +func (connector *FileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { + meshData := new(meshdata.MeshData) + + jsonFile, err := os.Open(connector.filePath) + if err != nil { + return nil, fmt.Errorf("unable to open input file '%v': %v", connector.filePath, err) + } + defer jsonFile.Close() + + jsonData, err := ioutil.ReadAll(jsonFile) + if err != nil { + return nil, fmt.Errorf("unable to read input file '%v': %v", connector.filePath, err) + } + + if err := json.Unmarshal(jsonData, &meshData.Sites); err != nil { + return nil, fmt.Errorf("invalid input file '%v': %v", connector.filePath, err) + } + + // Update the site types, as these are not part of the JSON data + connector.setSiteTypes(meshData) + + return meshData, nil +} + +func (connector *FileConnector) setSiteTypes(meshData *meshdata.MeshData) { + for _, site := range meshData.Sites { + site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites + } +} + +// GetName returns the display name of the connector. +func (connector *FileConnector) GetName() string { + return "File" +} + +func init() { + registerConnector(config.ConnectorIDFile, &FileConnector{}) +} diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 38784d7ab3..07daceed69 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -33,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/network" ) +// GOCDBConnector is used to read mesh data from a GOCDB instance. type GOCDBConnector struct { BaseConnector @@ -46,7 +47,7 @@ func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerol } // Check and store GOCDB specific settings - connector.gocdbAddress = conf.GOCDB.Address + connector.gocdbAddress = conf.Connectors.GOCDB.Address if len(connector.gocdbAddress) == 0 { return fmt.Errorf("no GOCDB address configured") } @@ -80,11 +81,11 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error { var scope string if hasScope { - scope = connector.conf.GOCDB.Scope + scope = connector.conf.Connectors.GOCDB.Scope } // Get the data from GOCDB - data, err := gocdb.QueryGOCDB(connector.conf.GOCDB.Address, method, isPrivate, scope, params) + data, err := gocdb.QueryGOCDB(connector.gocdbAddress, method, isPrivate, scope, params) if err != nil { return err } From b0c175818c0ac55e71c8764affc344b5da4029e1 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 10:51:17 +0100 Subject: [PATCH 03/35] Config improvements --- internal/http/services/mentix/mentix.go | 37 ++++++++----- pkg/mentix/config/config.go | 55 ++++++++++--------- pkg/mentix/config/ids.go | 4 +- pkg/mentix/connectors/gocdb.go | 4 +- .../connectors/{file.go => localfile.go} | 18 +++--- pkg/mentix/exporters/cs3api.go | 2 +- pkg/mentix/exporters/exporter.go | 2 +- pkg/mentix/exporters/promsd.go | 4 +- pkg/mentix/exporters/sitelocations.go | 2 +- pkg/mentix/exporters/webapi.go | 2 +- 10 files changed, 72 insertions(+), 58 deletions(-) rename pkg/mentix/connectors/{file.go => localfile.go} (79%) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 0ece4feda8..3bd00c15d7 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -27,7 +27,6 @@ import ( "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -88,10 +87,26 @@ func parseConfig(m map[string]interface{}) (*config.Configuration, error) { if err := mapstructure.Decode(m, &cfg); err != nil { return nil, errors.Wrap(err, "mentix: error decoding configuration") } + applyInternalConfig(m, cfg) applyDefaultConfig(cfg) return cfg, nil } +func applyInternalConfig(m map[string]interface{}, conf *config.Configuration) { + getSubsections := func(section string) []string { + subsections := make([]string, 0, 5) + if list, ok := m[section].(map[string]interface{}); ok { + for name, _ := range list { + subsections = append(subsections, name) + } + } + return subsections + } + + conf.EnabledConnectors = getSubsections("connectors") + conf.EnabledExporters = getSubsections("exporters") +} + func applyDefaultConfig(conf *config.Configuration) { if conf.Prefix == "" { conf.Prefix = serviceName @@ -101,28 +116,24 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Connector = config.ConnectorIDGOCDB // Use GOCDB } - if len(conf.Exporters) == 0 { - conf.Exporters = exporters.RegisteredExporterIDs() // Enable all exporters - } - if conf.UpdateInterval == "" { conf.UpdateInterval = "1h" // Update once per hour } - if conf.Connectors.GOCDB.Scope == "" { - conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future + if conf.GOCDB.Scope == "" { + conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future } - if conf.WebAPI.Endpoint == "" { - conf.WebAPI.Endpoint = "/" + if conf.Exporters.WebAPI.Endpoint == "" { + conf.Exporters.WebAPI.Endpoint = "/" } - if conf.CS3API.Endpoint == "" { - conf.CS3API.Endpoint = "/cs3" + if conf.Exporters.CS3API.Endpoint == "" { + conf.Exporters.CS3API.Endpoint = "/cs3" } - if conf.SiteLocations.Endpoint == "" { - conf.SiteLocations.Endpoint = "/loc" + if conf.Exporters.SiteLocations.Endpoint == "" { + conf.Exporters.SiteLocations.Endpoint = "/loc" } } diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 065a117cd4..8edf5b9cb2 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -22,40 +22,43 @@ package config type Configuration struct { Prefix string `mapstructure:"prefix"` - Connector string `mapstructure:"connector"` - Exporters []string `mapstructure:"exporters"` - UpdateInterval string `mapstructure:"update_interval"` + Connector string `mapstructure:"connector"` + UpdateInterval string `mapstructure:"update_interval"` - Connectors struct { - GOCDB struct { - Address string `mapstructure:"address"` - Scope string `mapstructure:"scope"` - } `mapstructure:"gocdb"` + GOCDB struct { + Address string `mapstructure:"address"` + Scope string `mapstructure:"scope"` + } `mapstructure:"gocdb"` - File struct { - File string `mapstructure:"file"` - } `mapstructure:"file"` - } `mapstructure:"connectors"` + LocalFile struct { + File string `mapstructure:"file"` + } `mapstructure:"localfile"` - WebAPI struct { - Endpoint string `mapstructure:"endpoint"` - } `yaml:"webapi"` + Exporters struct { + WebAPI struct { + Endpoint string `mapstructure:"endpoint"` + } `mapstructure:"webapi"` - CS3API struct { - Endpoint string `mapstructure:"endpoint"` - } `yaml:"cs3api"` + CS3API struct { + Endpoint string `mapstructure:"endpoint"` + } `mapstructure:"cs3api"` - SiteLocations struct { - Endpoint string `mapstructure:"endpoint"` - } `yaml:"siteloc"` + SiteLocations struct { + Endpoint string `mapstructure:"endpoint"` + } `mapstructure:"siteloc"` - PrometheusSD struct { - MetricsOutputFile string `mapstructure:"metrics_output_file"` - BlackboxOutputFile string `mapstructure:"blackbox_output_file"` - } `mapstructure:"promsd"` + PrometheusSD struct { + MetricsOutputFile string `mapstructure:"metrics_output_file"` + BlackboxOutputFile string `mapstructure:"blackbox_output_file"` + } `mapstructure:"promsd"` + } `mapstructure:"exporters"` + + // Internal settings + EnabledConnectors []string `mapstructure:"-"` + EnabledExporters []string `mapstructure:"-"` } -// Init sets sane defaults +// Init sets sane defaults. func (c *Configuration) Init() { if c.Prefix == "" { c.Prefix = "mentix" diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 5432d4b6fc..f89f411fb0 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -21,8 +21,8 @@ package config const ( // ConnectorIDGOCDB is the connector identifier for GOCDB. ConnectorIDGOCDB = "gocdb" - // ConnectorIDFile is the connector identifier for local files. - ConnectorIDFile = "file" + // ConnectorIDLocalFile is the connector identifier for local files. + ConnectorIDLocalFile = "localfile" ) const ( diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 07daceed69..92ec6f7474 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -47,7 +47,7 @@ func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerol } // Check and store GOCDB specific settings - connector.gocdbAddress = conf.Connectors.GOCDB.Address + connector.gocdbAddress = conf.GOCDB.Address if len(connector.gocdbAddress) == 0 { return fmt.Errorf("no GOCDB address configured") } @@ -81,7 +81,7 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error { var scope string if hasScope { - scope = connector.conf.Connectors.GOCDB.Scope + scope = connector.conf.GOCDB.Scope } // Get the data from GOCDB diff --git a/pkg/mentix/connectors/file.go b/pkg/mentix/connectors/localfile.go similarity index 79% rename from pkg/mentix/connectors/file.go rename to pkg/mentix/connectors/localfile.go index fefb7b1a82..e4a5f19317 100755 --- a/pkg/mentix/connectors/file.go +++ b/pkg/mentix/connectors/localfile.go @@ -30,21 +30,21 @@ import ( "github.com/cs3org/reva/pkg/mentix/meshdata" ) -// FileConnector is used to read sites from a local file. -type FileConnector struct { +// LocalFileConnector is used to read sites from a local file. +type LocalFileConnector struct { BaseConnector filePath string } // Activate activates the connector. -func (connector *FileConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { +func (connector *LocalFileConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if err := connector.BaseConnector.Activate(conf, log); err != nil { return err } // Check and store GOCDB specific settings - connector.filePath = conf.Connectors.File.File + connector.filePath = conf.LocalFile.File if len(connector.filePath) == 0 { return fmt.Errorf("no file configured") } @@ -53,7 +53,7 @@ func (connector *FileConnector) Activate(conf *config.Configuration, log *zerolo } // RetrieveMeshData fetches new mesh data. -func (connector *FileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { +func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { meshData := new(meshdata.MeshData) jsonFile, err := os.Open(connector.filePath) @@ -77,17 +77,17 @@ func (connector *FileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { return meshData, nil } -func (connector *FileConnector) setSiteTypes(meshData *meshdata.MeshData) { +func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { for _, site := range meshData.Sites { site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites } } // GetName returns the display name of the connector. -func (connector *FileConnector) GetName() string { - return "File" +func (connector *LocalFileConnector) GetName() string { + return "Local file" } func init() { - registerConnector(config.ConnectorIDFile, &FileConnector{}) + registerConnector(config.ConnectorIDLocalFile, &LocalFileConnector{}) } diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go index 38e946104e..54fe2e4b3d 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exporters/cs3api.go @@ -37,7 +37,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo } // Store CS3API specifics - exporter.endpoint = conf.CS3API.Endpoint + exporter.endpoint = conf.Exporters.CS3API.Endpoint exporter.defaultMethodHandler = cs3api.HandleDefaultQuery return nil diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index 71a792ef82..c393650027 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -113,7 +113,7 @@ func registerExporter(id string, exporter Exporter) { func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { // Try to add all exporters configured in the environment var exporters []Exporter - for _, exporterID := range conf.Exporters { + for _, exporterID := range conf.EnabledExporters { if exporter, ok := registeredExporters[exporterID]; ok { exporters = append(exporters, exporter) } else { diff --git a/pkg/mentix/exporters/promsd.go b/pkg/mentix/exporters/promsd.go index 9d68b16f39..2c4fd83369 100755 --- a/pkg/mentix/exporters/promsd.go +++ b/pkg/mentix/exporters/promsd.go @@ -102,11 +102,11 @@ func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Config } // Register all scrape creators - if err := registerCreator("metrics", conf.PrometheusSD.MetricsOutputFile, createMetricsSDScrapeConfig); err != nil { + if err := registerCreator("metrics", conf.Exporters.PrometheusSD.MetricsOutputFile, createMetricsSDScrapeConfig); err != nil { return fmt.Errorf("unable to register the 'metrics' scrape config creator: %v", err) } - if err := registerCreator("blackbox", conf.PrometheusSD.BlackboxOutputFile, createBlackboxSDScrapeConfig); err != nil { + if err := registerCreator("blackbox", conf.Exporters.PrometheusSD.BlackboxOutputFile, createBlackboxSDScrapeConfig); err != nil { return fmt.Errorf("unable to register the 'blackbox' scrape config creator: %v", err) } diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exporters/sitelocations.go index dde1fd5d69..d23ae86878 100755 --- a/pkg/mentix/exporters/sitelocations.go +++ b/pkg/mentix/exporters/sitelocations.go @@ -37,7 +37,7 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log } // Store SiteLocations specifics - exporter.endpoint = conf.SiteLocations.Endpoint + exporter.endpoint = conf.Exporters.SiteLocations.Endpoint exporter.defaultMethodHandler = siteloc.HandleDefaultQuery return nil diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index 71ff2c436e..2e429481c6 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -37,7 +37,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo } // Store WebAPI specifics - exporter.endpoint = conf.WebAPI.Endpoint + exporter.endpoint = conf.Exporters.WebAPI.Endpoint exporter.defaultMethodHandler = webapi.HandleDefaultQuery return nil From 776a1957387168a2c5d74615be2aa92628a75f0a Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 11:21:41 +0100 Subject: [PATCH 04/35] Support multiple connectors --- internal/http/services/mentix/mentix.go | 10 ++--- pkg/mentix/config/config.go | 19 ++++---- pkg/mentix/connectors/connector.go | 22 +++++++--- pkg/mentix/connectors/gocdb.go | 4 +- pkg/mentix/connectors/localfile.go | 2 +- pkg/mentix/mentix.go | 58 +++++++++++++++---------- pkg/mentix/meshdata/meshdata.go | 6 +++ 7 files changed, 72 insertions(+), 49 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 3bd00c15d7..8e2f6ec8c7 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -96,7 +96,7 @@ func applyInternalConfig(m map[string]interface{}, conf *config.Configuration) { getSubsections := func(section string) []string { subsections := make([]string, 0, 5) if list, ok := m[section].(map[string]interface{}); ok { - for name, _ := range list { + for name := range list { subsections = append(subsections, name) } } @@ -112,16 +112,12 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Prefix = serviceName } - if conf.Connector == "" { - conf.Connector = config.ConnectorIDGOCDB // Use GOCDB - } - if conf.UpdateInterval == "" { conf.UpdateInterval = "1h" // Update once per hour } - if conf.GOCDB.Scope == "" { - conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future + if conf.Connectors.GOCDB.Scope == "" { + conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future } if conf.Exporters.WebAPI.Endpoint == "" { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 8edf5b9cb2..b321cbbcad 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -22,17 +22,18 @@ package config type Configuration struct { Prefix string `mapstructure:"prefix"` - Connector string `mapstructure:"connector"` - UpdateInterval string `mapstructure:"update_interval"` + Connectors struct { + GOCDB struct { + Address string `mapstructure:"address"` + Scope string `mapstructure:"scope"` + } `mapstructure:"gocdb"` - GOCDB struct { - Address string `mapstructure:"address"` - Scope string `mapstructure:"scope"` - } `mapstructure:"gocdb"` + LocalFile struct { + File string `mapstructure:"file"` + } `mapstructure:"localfile"` + } `mapstructure:"connectors"` - LocalFile struct { - File string `mapstructure:"file"` - } `mapstructure:"localfile"` + UpdateInterval string `mapstructure:"update_interval"` Exporters struct { WebAPI struct { diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 635ca0344a..3dc22e177d 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -20,7 +20,6 @@ package connectors import ( "fmt" - "strings" "github.com/rs/zerolog" @@ -64,15 +63,24 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo return nil } -// FindConnector searches for the given connector ID in all globally registered connectors. -func FindConnector(connectorID string) (Connector, error) { - for id, connector := range registeredConnectors { - if strings.EqualFold(id, connectorID) { - return connector, nil +// AvailableConnectors returns a list of all connectors that are enabled in the configuration. +func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { + // Try to add all connectors configured in the environment + var connectors []Connector + for _, connectorID := range conf.EnabledConnectors { + if connector, ok := registeredConnectors[connectorID]; ok { + connectors = append(connectors, connector) + } else { + return nil, fmt.Errorf("no connector with ID '%v' registered", connectorID) } } - return nil, fmt.Errorf("no connector with ID '%v' registered", connectorID) + // At least one connector must be configured + if len(connectors) == 0 { + return nil, fmt.Errorf("no connectors available") + } + + return connectors, nil } func registerConnector(id string, connector Connector) { diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 92ec6f7474..07daceed69 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -47,7 +47,7 @@ func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerol } // Check and store GOCDB specific settings - connector.gocdbAddress = conf.GOCDB.Address + connector.gocdbAddress = conf.Connectors.GOCDB.Address if len(connector.gocdbAddress) == 0 { return fmt.Errorf("no GOCDB address configured") } @@ -81,7 +81,7 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error { var scope string if hasScope { - scope = connector.conf.GOCDB.Scope + scope = connector.conf.Connectors.GOCDB.Scope } // Get the data from GOCDB diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index e4a5f19317..8323f10b0f 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -44,7 +44,7 @@ func (connector *LocalFileConnector) Activate(conf *config.Configuration, log *z } // Check and store GOCDB specific settings - connector.filePath = conf.LocalFile.File + connector.filePath = conf.Connectors.LocalFile.File if len(connector.filePath) == 0 { return fmt.Errorf("no file configured") } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 4b0e4eabeb..7162f1e039 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -38,9 +38,9 @@ type Mentix struct { conf *config.Configuration log *zerolog.Logger - meshData *meshdata.MeshData - connector connectors.Connector - exporters []exporters.Exporter + meshData *meshdata.MeshData + connectors []connectors.Connector + exporters []exporters.Exporter updateInterval time.Duration } @@ -60,8 +60,8 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) } mntx.log = log - // Initialize the connector that will be used to gather the mesh data - if err := mntx.initConnector(); err != nil { + // Initialize the connectors that will be used to gather the mesh data + if err := mntx.initConnectors(); err != nil { return fmt.Errorf("unable to initialize connector: %v", err) } @@ -82,40 +82,46 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) mntx.meshData = meshdata.New() // Log some infos + connectorNames := make([]string, len(mntx.connectors)) + for idx, connector := range mntx.connectors { + connectorNames[idx] = connector.GetName() + } exporterNames := make([]string, len(mntx.exporters)) for idx, exporter := range mntx.exporters { exporterNames[idx] = exporter.GetName() } - log.Info().Msgf("mentix started with connector: %v; exporters: %v; update interval: %v", mntx.connector.GetName(), strings.Join(exporterNames, ", "), duration) + log.Info().Msgf("mentix started with connectors: %v; exporters: %v; update interval: %v", strings.Join(connectorNames, ", "), strings.Join(exporterNames, ", "), duration) return nil } -func (mntx *Mentix) initConnector() error { - // Try to get a connector with the configured ID - connector, err := connectors.FindConnector(mntx.conf.Connector) +func (mntx *Mentix) initConnectors() error { + // Use all conns exposed by the conns package + conns, err := connectors.AvailableConnectors(mntx.conf) if err != nil { - return fmt.Errorf("the desired connector could be found: %v", err) + return fmt.Errorf("unable to get registered conns: %v", err) } - mntx.connector = connector + mntx.connectors = conns - // Activate the selected connector - if err := mntx.connector.Activate(mntx.conf, mntx.log); err != nil { - return fmt.Errorf("unable to activate connector: %v", err) + // Activate all conns + for _, connector := range mntx.connectors { + if err := connector.Activate(mntx.conf, mntx.log); err != nil { + return fmt.Errorf("unable to activate connector '%v': %v", connector.GetName(), err) + } } return nil } func (mntx *Mentix) initExporters() error { - // Use all exporters exposed by the exporters package - exporters, err := exporters.AvailableExporters(mntx.conf) + // Use all exps exposed by the exps package + exps, err := exporters.AvailableExporters(mntx.conf) if err != nil { - return fmt.Errorf("unable to get registered exporters: %v", err) + return fmt.Errorf("unable to get registered exps: %v", err) } - mntx.exporters = exporters + mntx.exporters = exps - // Activate all exporters + // Activate all exps for _, exporter := range mntx.exporters { if err := exporter.Activate(mntx.conf, mntx.log); err != nil { return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) @@ -192,11 +198,17 @@ loop: } func (mntx *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { - meshData, err := mntx.connector.RetrieveMeshData() - if err != nil { - return nil, fmt.Errorf("retrieving mesh data failed: %v", err) + var mergedMeshData meshdata.MeshData + + for _, connector := range mntx.connectors { + meshData, err := connector.RetrieveMeshData() + if err != nil { + return nil, fmt.Errorf("retrieving mesh data from connector '%v' failed: %v", connector.GetName(), err) + } + mergedMeshData.Merge(meshData) } - return meshData, nil + + return &mergedMeshData, nil } func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 1d593c523e..46d4aa1179 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -35,6 +35,12 @@ func (meshData *MeshData) Clear() { meshData.ServiceTypes = nil } +// Merge merges data from another MeshData instance into this one (w/o checking for duplicates). +func (meshData *MeshData) Merge(inData *MeshData) { + meshData.Sites = append(meshData.Sites, inData.Sites...) + meshData.ServiceTypes = append(meshData.ServiceTypes, inData.ServiceTypes...) +} + // ToJSON converts the data to JSON. func (meshData *MeshData) ToJSON() (string, error) { data, err := json.MarshalIndent(meshData, "", "\t") From cb59dca4652c0e1417273a937af588b4b0f8d547 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 12:10:00 +0100 Subject: [PATCH 05/35] Adjust example to new configuration structure --- examples/mentix/mentix.toml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 5ea0c8d8e8..f5d5d950ad 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -6,26 +6,20 @@ address = "0.0.0.0:9600" enabled_services = ["mentix"] [http.services.mentix] -connector = "gocdb" -exporters = ["webapi", "cs3api", "siteloc"] -# Enable the Prometheus Service Discovery: -# exporters = ["webapi", "cs3api", "siteloc", "promsd"] update_interval = "15m" -[http.services.mentix.gocdb] +[http.services.mentix.connectors.gocdb] address = "http://sciencemesh-test.uni-muenster.de" -[http.services.mentix.webapi] +[http.services.mentix.connectors.webapi] endpoint = "/" -[http.services.mentix.cs3api] -endpoint = "/cs3" - -[http.services.mentix.siteloc] -endpoint = "/loc" +# Enable the CS3API and Site Locations exporters +[http.services.mentix.exporters.cs3api] +[http.services.mentix.exporters.siteloc] # Configure the Prometheus Service Discovery: -# [http.services.mentix.promsd] +# [http.services.mentix.exporters.promsd] # The following files must be made available to Prometheus. # They can then be used as the file_sd source of a job. # metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json" From 09de6df562c54285701eeb1d1bd4372fba9bd640 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 12:29:40 +0100 Subject: [PATCH 06/35] Update documentation --- .../config/http/services/mentix/_index.md | 43 ++++++++----------- .../http/services/mentix/cs3api/_index.md | 2 +- .../http/services/mentix/gocdb/_index.md | 4 +- .../http/services/mentix/localfile/_index.md | 19 ++++++++ .../http/services/mentix/promsd/_index.md | 4 +- .../http/services/mentix/siteloc/_index.md | 2 +- .../http/services/mentix/webapi/_index.md | 2 +- 7 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 docs/content/en/docs/config/http/services/mentix/localfile/_index.md diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 90944cdc91..de944fad3c 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -7,7 +7,7 @@ description: > --- {{% pageinfo %}} -Mentix (_**Me**sh E**nti**ty E**x**porter_) is a service to read mesh topology data from a source (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. +Mentix (_**Me**sh E**nti**ty E**x**porter_) is a service to read mesh topology data from one or more sources (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. {{% /pageinfo %}} {{% dir name="prefix" type="string" default="mentix" %}} @@ -18,24 +18,29 @@ prefix = "/mentix" {{< /highlight >}} {{% /dir %}} -{{% dir name="connector" type="string" default="gocdb" %}} -Mentix is decoupled from the actual source of the mesh data by using a so-called _connector_. A connector is used to gather the data from a certain source, which are then converted into Mentix' own internal format. - -Supported values are: - -- **gocdb** -The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)). - +{{% dir name="update_interval" type="string" default="1h" %}} +How frequently Mentix should pull and update the mesh data. Supports common time duration strings, like "1h30m", "1d" etc. {{< highlight toml >}} [http.services.mentix] -connector = "gocdb" +update_interval = "15m" {{< /highlight >}} {{% /dir %}} -{{% dir name="exporters" type="[]string" default="[webapi,cs3api,siteloc,promsd]" %}} +#### Connectors +Mentix is decoupled from the actual sources of the mesh data by using so-called _connectors_. A connector is used to gather the data from a certain source, which are then converted into Mentix' own internal format. + +_Supported connectors:_ + +- **gocdb** +The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)). + +- **localfile** +The [localfile](localfile) connector reads sites from a local JSON file. The file must contain an array of sites adhering to the `meshdata.Site` structure. + +#### Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. -Supported values are: +__Supported exporters:__ - **webapi** Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. @@ -49,17 +54,3 @@ Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data c - files: - '/usr/share/prom/sciencemesh_services.json' ``` - -{{< highlight toml >}} -[http.services.mentix] -exporters = ["webapi", "promsd"] -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="update_interval" type="string" default="1h" %}} -How frequently Mentix should pull and update the mesh data. Supports common time duration strings, like "1h30m", "1d" etc. -{{< highlight toml >}} -[http.services.mentix] -update_interval = "15m" -{{< /highlight >}} -{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md index e89b810c4e..da86082dee 100644 --- a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md @@ -13,7 +13,7 @@ The CS3API exporter exposes Mentix data in a format that is compliant with the C {{% dir name="endpoint" type="string" default="/" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} -[http.services.mentix.cs3api] +[http.services.mentix.exporters.cs3api] endpoint = "/data" {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/gocdb/_index.md b/docs/content/en/docs/config/http/services/mentix/gocdb/_index.md index 4ae7cdf81f..f84a3ae415 100644 --- a/docs/content/en/docs/config/http/services/mentix/gocdb/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/gocdb/_index.md @@ -13,7 +13,7 @@ When using the [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) conne {{% dir name="address" type="string" default="" %}} The address of the GOCDB instance; must be a valid URL (e.g., http://gocdb.uni-muenster.de). **Note:** The public API must be reachable under `
/gocdbpi/public`. {{< highlight toml >}} -[http.services.mentix.gocdb] +[http.services.mentix.connectors.gocdb] address = "http://gocdb.example.com" {{< /highlight >}} {{% /dir %}} @@ -21,7 +21,7 @@ address = "http://gocdb.example.com" {{% dir name="scope" type="string" default="SM" %}} The scope to use for filtering sites and services. {{< highlight toml >}} -[http.services.mentix.gocdb] +[http.services.mentix.connectors.gocdb] scope = "SM" {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/localfile/_index.md b/docs/content/en/docs/config/http/services/mentix/localfile/_index.md new file mode 100644 index 0000000000..0a85c0059a --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/localfile/_index.md @@ -0,0 +1,19 @@ +--- +title: "localfile" +linkTitle: "localfile" +weight: 10 +description: > + Configuration for the local file connector of the Mentix service +--- + +{{% pageinfo %}} +The local file connector reads sites from a local JSON file adhering to the `meshdata.Site` structure. +{{% /pageinfo %}} + +{{% dir name="file" type="string" default="" %}} +The file path. +{{< highlight toml >}} +[http.services.mentix.connectors.localfile] +file = "/etc/reva/sites.json" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md index 58375f7449..0b69756af3 100644 --- a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md @@ -13,7 +13,7 @@ When using the Prometheus SD exporter, the output filenames have to be configure {{% dir name="metrics_output_file" type="string" default="" %}} The target filename of the generated Prometheus File SD scrape config for metrics. {{< highlight toml >}} -[http.services.mentix.promsd] +[http.services.mentix.exporters.promsd] metrics_output_file = "/var/shared/prometheus/sciencemesh.json" {{< /highlight >}} {{% /dir %}} @@ -21,7 +21,7 @@ metrics_output_file = "/var/shared/prometheus/sciencemesh.json" {{% dir name="blackbox_output_file" type="string" default="" %}} The target filename of the generated Prometheus File SD scrape config for the blackbox exporter. {{< highlight toml >}} -[http.services.mentix.promsd] +[http.services.mentix.exporters.promsd] blackbox_output_file = "/var/shared/prometheus/blackbox.json" {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md index 1e71c66fbc..5ccc48c613 100644 --- a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -13,7 +13,7 @@ The Site Locations exporter exposes location information of all sites to be cons {{% dir name="endpoint" type="string" default="/" %}} The endpoint where the locations data can be queried. {{< highlight toml >}} -[http.services.mentix.siteloc] +[http.services.mentix.exporters.siteloc] endpoint = "/loc" {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index 5c190f773b..edfaa75668 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -13,7 +13,7 @@ The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. {{% dir name="endpoint" type="string" default="/" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} -[http.services.mentix.webapi] +[http.services.mentix.exporters.webapi] endpoint = "/data" {{< /highlight >}} {{% /dir %}} From 9862f07fec930ee499a2bf27f870983519eb5f8b Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 16:20:21 +0100 Subject: [PATCH 07/35] Move lots of shared functionality into 'exchange' package --- pkg/mentix/connectors/connector.go | 19 +++-- pkg/mentix/exchange/exchanger.go | 114 ++++++++++++++++++++++++++ pkg/mentix/exchange/registry.go | 51 ++++++++++++ pkg/mentix/exchange/reqexchanger.go | 66 +++++++++++++++ pkg/mentix/exporters/cs3api.go | 4 +- pkg/mentix/exporters/exporter.go | 81 ++++-------------- pkg/mentix/exporters/promsd.go | 12 +-- pkg/mentix/exporters/reqexporter.go | 45 ++-------- pkg/mentix/exporters/sitelocations.go | 4 +- pkg/mentix/exporters/webapi.go | 4 +- pkg/mentix/mentix.go | 30 +++++-- 11 files changed, 302 insertions(+), 128 deletions(-) create mode 100644 pkg/mentix/exchange/exchanger.go create mode 100644 pkg/mentix/exchange/registry.go create mode 100644 pkg/mentix/exchange/reqexchanger.go diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 3dc22e177d..fb4fead693 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -24,11 +24,12 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" ) var ( - registeredConnectors = map[string]Connector{} + registeredConnectors = exchange.NewRegistry() ) // Connector is the interface that all connectors must implement. @@ -65,14 +66,14 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo // AvailableConnectors returns a list of all connectors that are enabled in the configuration. func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { - // Try to add all connectors configured in the environment + entries, err := registeredConnectors.EntriesByID(conf.EnabledConnectors) + if err != nil { + return nil, err + } + var connectors []Connector - for _, connectorID := range conf.EnabledConnectors { - if connector, ok := registeredConnectors[connectorID]; ok { - connectors = append(connectors, connector) - } else { - return nil, fmt.Errorf("no connector with ID '%v' registered", connectorID) - } + for _, entry := range entries { + connectors = append(connectors, entry.(Connector)) } // At least one connector must be configured @@ -84,5 +85,5 @@ func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { } func registerConnector(id string, connector Connector) { - registeredConnectors[id] = connector + registeredConnectors.Register(id, connector) } diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchange/exchanger.go new file mode 100644 index 0000000000..ed550b674c --- /dev/null +++ b/pkg/mentix/exchange/exchanger.go @@ -0,0 +1,114 @@ +/* + * MIT License + * + * Copyright (c) 2020 Daniel Mueller + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package exchange + +import ( + "fmt" + "sync" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +// Exchanger is the base interface for importers and exporters. +type Exchanger interface { + // Activate activates the exchanger. + Activate(conf *config.Configuration, log *zerolog.Logger) error + // Start starts the exchanger; only exchangers which perform periodical background tasks should do something here. + Start() error + // Stop stops any running background activities of the exchanger. + Stop() + + // MeshData returns the mesh data object. + MeshData() *meshdata.MeshData + + // GetName returns the display name of the exchanger. + GetName() string +} + +// BaseExchanger implements basic exchanger functionality common to all exchangers. +type BaseExchanger struct { + Exchanger + + conf *config.Configuration + log *zerolog.Logger + + meshData *meshdata.MeshData + locker sync.RWMutex +} + +// Activate activates the exchanger. +func (exchanger *BaseExchanger) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if conf == nil { + return fmt.Errorf("no configuration provided") + } + exchanger.conf = conf + + if log == nil { + return fmt.Errorf("no logger provided") + } + exchanger.log = log + + return nil +} + +// Start starts the exchanger; only exchanger which perform periodical background tasks should do something here. +func (exchanger *BaseExchanger) Start() error { + return nil +} + +// Stop stops any running background activities of the exchanger. +func (exchanger *BaseExchanger) Stop() { +} + +// Config returns the configuration object. +func (exchanger *BaseExchanger) Config() *config.Configuration { + return exchanger.conf +} + +// Log returns the logger object. +func (exchanger *BaseExchanger) Log() *zerolog.Logger { + return exchanger.log +} + +// MeshData returns the mesh data object. +func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { + return exchanger.meshData +} + +// SetMeshData sets new mesh data. +func (exchanger *BaseExchanger) SetMeshData(meshData *meshdata.MeshData) { + exchanger.Locker().Lock() + defer exchanger.Locker().Unlock() + + exchanger.meshData = meshData +} + +// Locker returns the locking object. +func (exchanger *BaseExchanger) Locker() *sync.RWMutex { + return &exchanger.locker +} diff --git a/pkg/mentix/exchange/registry.go b/pkg/mentix/exchange/registry.go new file mode 100644 index 0000000000..bf1b763373 --- /dev/null +++ b/pkg/mentix/exchange/registry.go @@ -0,0 +1,51 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exchange + +import "fmt" + +type RegistryEntry interface { +} + +type Registry struct { + Entries map[string]RegistryEntry +} + +func (r *Registry) Register(id string, entry RegistryEntry) { + r.Entries[id] = entry +} + +func (r *Registry) EntriesByID(ids []string) ([]RegistryEntry, error) { + var entries []RegistryEntry + for _, id := range ids { + if entry, ok := r.Entries[id]; ok { + entries = append(entries, entry) + } else { + return nil, fmt.Errorf("no entry with ID '%v' registered", id) + } + } + + return entries, nil +} + +func NewRegistry() *Registry { + return &Registry{ + Entries: make(map[string]RegistryEntry), + } +} diff --git a/pkg/mentix/exchange/reqexchanger.go b/pkg/mentix/exchange/reqexchanger.go new file mode 100644 index 0000000000..5322190adf --- /dev/null +++ b/pkg/mentix/exchange/reqexchanger.go @@ -0,0 +1,66 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exchange + +import ( + "net/http" + "strings" +) + +// RequestExchanger is the interface implemented by exchangers that offer an HTTP endpoint. +type RequestExchanger interface { + // Endpoint returns the (relative) endpoint of the exchanger. + Endpoint() string + // WantsRequest returns whether the exchanger wants to handle the incoming request. + WantsRequest(r *http.Request) bool + // HandleRequest handles the actual HTTP request. + HandleRequest(resp http.ResponseWriter, req *http.Request) error +} + +// BaseRequestExchanger implements basic exporter functionality common to all request exporters. +type BaseRequestExchanger struct { + RequestExchanger + + endpoint string +} + +// Endpoint returns the (relative) endpoint of the exchanger. +func (exchanger *BaseRequestExchanger) Endpoint() string { + // Ensure that the endpoint starts with a / + endpoint := exchanger.endpoint + if !strings.HasPrefix(endpoint, "/") { + endpoint = "/" + endpoint + } + return strings.TrimSpace(endpoint) +} + +// SetEndpoint sets the (relative) endpoint of the exchanger. +func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string) { + exchanger.endpoint = endpoint +} + +// WantsRequest returns whether the exporter wants to handle the incoming request. +func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool { + return r.URL.Path == exchanger.Endpoint() +} + +// HandleRequest handles the actual HTTP request. +func (exchanger *BaseRequestExchanger) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + return nil +} diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exporters/cs3api.go index 54fe2e4b3d..c44bce4b9a 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exporters/cs3api.go @@ -32,12 +32,12 @@ type CS3APIExporter struct { // Activate activates the exporter. func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseExporter.Activate(conf, log); err != nil { + if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil { return err } // Store CS3API specifics - exporter.endpoint = conf.Exporters.CS3API.Endpoint + exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint) exporter.defaultMethodHandler = cs3api.HandleDefaultQuery return nil diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index c393650027..96f69080f7 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -20,66 +20,27 @@ package exporters import ( "fmt" - "sync" - - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" ) var ( - registeredExporters = map[string]Exporter{} + registeredExporters = exchange.NewRegistry() ) // Exporter is the interface that all exporters must implement. type Exporter interface { - // Activate activates the exporter. - Activate(conf *config.Configuration, log *zerolog.Logger) error - // Start starts the exporter; only exporters which perform periodical background tasks should do something here. - Start() error - // Stop stops any running background activities of the exporter. - Stop() + exchange.Exchanger // UpdateMeshData is called whenever the mesh data has changed to reflect these changes. UpdateMeshData(*meshdata.MeshData) error - - // GetName returns the display name of the exporter. - GetName() string } // BaseExporter implements basic exporter functionality common to all exporters. type BaseExporter struct { - conf *config.Configuration - log *zerolog.Logger - - meshData *meshdata.MeshData - locker sync.RWMutex -} - -// Activate activates the exporter. -func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if conf == nil { - return fmt.Errorf("no configuration provided") - } - exporter.conf = conf - - if log == nil { - return fmt.Errorf("no logger provided") - } - exporter.log = log - - return nil -} - -// Start starts the exporter; only exporters which perform periodical background tasks should do something here. -func (exporter *BaseExporter) Start() error { - return nil -} - -// Stop stops any running background activities of the exporter. -func (exporter *BaseExporter) Stop() { - + exchange.BaseExchanger } // UpdateMeshData is called whenever the mesh data has changed to reflect these changes. @@ -93,32 +54,27 @@ func (exporter *BaseExporter) UpdateMeshData(meshData *meshdata.MeshData) error } func (exporter *BaseExporter) storeMeshData(meshData *meshdata.MeshData) error { - exporter.locker.Lock() - defer exporter.locker.Unlock() - // Store the new mesh data by cloning it - exporter.meshData = meshData.Clone() - if exporter.meshData == nil { + meshDataCloned := meshData.Clone() + if meshDataCloned == nil { return fmt.Errorf("unable to clone the mesh data") } + exporter.SetMeshData(meshDataCloned) return nil } -func registerExporter(id string, exporter Exporter) { - registeredExporters[id] = exporter -} - // AvailableExporters returns a list of all exporters that are enabled in the configuration. func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { // Try to add all exporters configured in the environment + entries, err := registeredExporters.EntriesByID(conf.EnabledExporters) + if err != nil { + return nil, err + } + var exporters []Exporter - for _, exporterID := range conf.EnabledExporters { - if exporter, ok := registeredExporters[exporterID]; ok { - exporters = append(exporters, exporter) - } else { - return nil, fmt.Errorf("no exporter with ID '%v' registered", exporterID) - } + for _, entry := range entries { + exporters = append(exporters, entry.(Exporter)) } // At least one exporter must be configured @@ -129,11 +85,6 @@ func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { return exporters, nil } -// RegisteredExporterIDs returns a list of all registered exporter IDs. -func RegisteredExporterIDs() []string { - keys := make([]string, 0, len(registeredExporters)) - for k := range registeredExporters { - keys = append(keys, k) - } - return keys +func registerExporter(id string, exporter Exporter) { + registeredExporters.Register(id, exporter) } diff --git a/pkg/mentix/exporters/promsd.go b/pkg/mentix/exporters/promsd.go index 2c4fd83369..d1fc753412 100755 --- a/pkg/mentix/exporters/promsd.go +++ b/pkg/mentix/exporters/promsd.go @@ -146,15 +146,15 @@ func (exporter *PrometheusSDExporter) UpdateMeshData(meshData *meshdata.MeshData func (exporter *PrometheusSDExporter) exportMeshData() { // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() + exporter.Locker().RLock() + defer exporter.Locker().RUnlock() for name, creator := range exporter.scrapeCreators { scrapes := exporter.createScrapeConfigs(creator.creatorCallback) if err := exporter.exportScrapeConfig(creator.outputFilename, scrapes); err != nil { - exporter.log.Err(err).Str("kind", name).Str("file", creator.outputFilename).Msg("error exporting Prometheus SD scrape config") + exporter.Log().Err(err).Str("kind", name).Str("file", creator.outputFilename).Msg("error exporting Prometheus SD scrape config") } else { - exporter.log.Debug().Str("kind", name).Str("file", creator.outputFilename).Msg("exported Prometheus SD scrape config") + exporter.Log().Debug().Str("kind", name).Str("file", creator.outputFilename).Msg("exported Prometheus SD scrape config") } } } @@ -168,7 +168,7 @@ func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback promet } // Create a scrape config for each service alongside any additional endpoints - for _, site := range exporter.meshData.Sites { + for _, site := range exporter.MeshData().Sites { for _, service := range site.Services { if !service.IsMonitored { continue @@ -206,7 +206,7 @@ func (exporter *PrometheusSDExporter) exportScrapeConfig(outputFilename string, // GetName returns the display name of the exporter. func (exporter *PrometheusSDExporter) GetName() string { - return "PrometheusSD SD" + return "Prometheus SD" } func init() { diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 4091952c08..a4804bf205 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -24,6 +24,7 @@ import ( "net/url" "strings" + "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -33,49 +34,17 @@ const ( type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) -// RequestExporter is the interface implemented by exporters that offer an HTTP endpoint. -type RequestExporter interface { - Exporter - - // Endpoint returns the (relative) endpoint of the exporter. - Endpoint() string - // WantsRequest returns whether the exporter wants to handle the incoming request. - WantsRequest(r *http.Request) bool - // HandleRequest handles the actual HTTP request. - HandleRequest(resp http.ResponseWriter, req *http.Request) error -} - // BaseRequestExporter implements basic exporter functionality common to all request exporters. type BaseRequestExporter struct { BaseExporter - - endpoint string + exchange.BaseRequestExchanger defaultMethodHandler queryCallback } -// Endpoint returns the (relative) endpoint of the exporter. -func (exporter *BaseRequestExporter) Endpoint() string { - // Ensure that the endpoint starts with a / - endpoint := exporter.endpoint - if !strings.HasPrefix(endpoint, "/") { - endpoint = "/" + endpoint - } - return strings.TrimSpace(endpoint) -} - -// WantsRequest returns whether the exporter wants to handle the incoming request. -func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { - return r.URL.Path == exporter.Endpoint() -} - // HandleRequest handles the actual HTTP request. func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - // Data is read, so acquire a read lock - exporter.locker.RLock() - defer exporter.locker.RUnlock() - - data, err := exporter.handleQuery(exporter.meshData, req.URL.Query()) + data, err := exporter.handleQuery(req.URL.Query()) if err == nil { if _, err := resp.Write(data); err != nil { return fmt.Errorf("error writing the API request response: %v", err) @@ -87,12 +56,16 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req return nil } -func (exporter *BaseRequestExporter) handleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +func (exporter *BaseRequestExporter) handleQuery(params url.Values) ([]byte, error) { + // Data is read, so lock it for writing + exporter.Locker().RLock() + defer exporter.Locker().RUnlock() + method := params.Get("method") switch strings.ToLower(method) { case queryMethodDefault: if exporter.defaultMethodHandler != nil { - return exporter.defaultMethodHandler(meshData, params) + return exporter.defaultMethodHandler(exporter.MeshData(), params) } default: diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exporters/sitelocations.go index d23ae86878..aa88c7c404 100755 --- a/pkg/mentix/exporters/sitelocations.go +++ b/pkg/mentix/exporters/sitelocations.go @@ -32,12 +32,12 @@ type SiteLocationsExporter struct { // Activate activates the exporter. func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseExporter.Activate(conf, log); err != nil { + if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil { return err } // Store SiteLocations specifics - exporter.endpoint = conf.Exporters.SiteLocations.Endpoint + exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint) exporter.defaultMethodHandler = siteloc.HandleDefaultQuery return nil diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index 2e429481c6..8a79fb38bc 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -32,12 +32,12 @@ type WebAPIExporter struct { // Activate activates the exporter. func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseExporter.Activate(conf, log); err != nil { + if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil { return err } // Store WebAPI specifics - exporter.endpoint = conf.Exporters.WebAPI.Endpoint + exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint) exporter.defaultMethodHandler = webapi.HandleDefaultQuery return nil diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 7162f1e039..fb72b926f9 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -29,6 +29,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" + "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/exporters" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -96,7 +97,7 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) } func (mntx *Mentix) initConnectors() error { - // Use all conns exposed by the conns package + // Use all connectors exposed by the connectors package conns, err := connectors.AvailableConnectors(mntx.conf) if err != nil { return fmt.Errorf("unable to get registered conns: %v", err) @@ -114,7 +115,7 @@ func (mntx *Mentix) initConnectors() error { } func (mntx *Mentix) initExporters() error { - // Use all exps exposed by the exps package + // Use all exporters exposed by the exporters package exps, err := exporters.AvailableExporters(mntx.conf) if err != nil { return fmt.Errorf("unable to get registered exps: %v", err) @@ -228,11 +229,11 @@ func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { } // GetRequestExporters returns all exporters that can handle HTTP requests. -func (mntx *Mentix) GetRequestExporters() []exporters.RequestExporter { +func (mntx *Mentix) GetRequestExporters() []exchange.RequestExchanger { // Return all exporters that implement the RequestExporter interface - var reqExporters []exporters.RequestExporter + var reqExporters []exchange.RequestExchanger for _, exporter := range mntx.exporters { - if reqExporter, ok := exporter.(exporters.RequestExporter); ok { + if reqExporter, ok := exporter.(exchange.RequestExchanger); ok { reqExporters = append(reqExporters, reqExporter) } } @@ -244,16 +245,33 @@ func (mntx *Mentix) GetRequestExporters() []exporters.RequestExporter { func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { log := appctx.GetLogger(r.Context()) + switch r.Method { + case http.MethodGet: + mntx.handleGetRequest(w, r, log) + + case http.MethodPost: + mntx.handlePostRequest(w, r, log) + + default: + log.Err(fmt.Errorf("unsupported method")).Msg("error handling incoming request") + } +} + +func (mntx *Mentix) handleGetRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { // Ask each RequestExporter if it wants to handle the request for _, exporter := range mntx.GetRequestExporters() { if exporter.WantsRequest(r) { if err := exporter.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling request") + log.Err(err).Msg("error handling GET request") } } } } +func (mntx *Mentix) handlePostRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { + +} + // New creates a new Mentix service instance. func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) { mntx := new(Mentix) From 71f5bb1aeadf7fdbb9e4def5d3285a0e7d83bda3 Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 16:27:01 +0100 Subject: [PATCH 08/35] Restructuring --- pkg/mentix/connectors/connector.go | 4 ++-- pkg/mentix/{ => exchange}/exporters/cs3api.go | 2 +- pkg/mentix/{ => exchange}/exporters/cs3api/query.go | 0 pkg/mentix/{ => exchange}/exporters/exporter.go | 3 ++- pkg/mentix/{ => exchange}/exporters/prometheus/types.go | 0 pkg/mentix/{ => exchange}/exporters/promsd.go | 2 +- pkg/mentix/{ => exchange}/exporters/reqexporter.go | 0 pkg/mentix/{ => exchange}/exporters/siteloc/query.go | 0 pkg/mentix/{ => exchange}/exporters/siteloc/types.go | 0 pkg/mentix/{ => exchange}/exporters/sitelocations.go | 2 +- pkg/mentix/{ => exchange}/exporters/webapi.go | 2 +- pkg/mentix/{ => exchange}/exporters/webapi/query.go | 0 pkg/mentix/mentix.go | 2 +- pkg/mentix/{exchange => registry}/registry.go | 2 +- 14 files changed, 10 insertions(+), 9 deletions(-) rename pkg/mentix/{ => exchange}/exporters/cs3api.go (96%) rename pkg/mentix/{ => exchange}/exporters/cs3api/query.go (100%) rename pkg/mentix/{ => exchange}/exporters/exporter.go (96%) rename pkg/mentix/{ => exchange}/exporters/prometheus/types.go (100%) rename pkg/mentix/{ => exchange}/exporters/promsd.go (99%) rename pkg/mentix/{ => exchange}/exporters/reqexporter.go (100%) rename pkg/mentix/{ => exchange}/exporters/siteloc/query.go (100%) rename pkg/mentix/{ => exchange}/exporters/siteloc/types.go (100%) rename pkg/mentix/{ => exchange}/exporters/sitelocations.go (96%) rename pkg/mentix/{ => exchange}/exporters/webapi.go (96%) rename pkg/mentix/{ => exchange}/exporters/webapi/query.go (100%) rename pkg/mentix/{exchange => registry}/registry.go (98%) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index fb4fead693..f293371291 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -24,12 +24,12 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/registry" ) var ( - registeredConnectors = exchange.NewRegistry() + registeredConnectors = registry.NewRegistry() ) // Connector is the interface that all connectors must implement. diff --git a/pkg/mentix/exporters/cs3api.go b/pkg/mentix/exchange/exporters/cs3api.go similarity index 96% rename from pkg/mentix/exporters/cs3api.go rename to pkg/mentix/exchange/exporters/cs3api.go index c44bce4b9a..3bbcca340f 100755 --- a/pkg/mentix/exporters/cs3api.go +++ b/pkg/mentix/exchange/exporters/cs3api.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters/cs3api" + "github.com/cs3org/reva/pkg/mentix/exchange/exporters/cs3api" ) // CS3APIExporter implements the CS3API exporter. diff --git a/pkg/mentix/exporters/cs3api/query.go b/pkg/mentix/exchange/exporters/cs3api/query.go similarity index 100% rename from pkg/mentix/exporters/cs3api/query.go rename to pkg/mentix/exchange/exporters/cs3api/query.go diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go similarity index 96% rename from pkg/mentix/exporters/exporter.go rename to pkg/mentix/exchange/exporters/exporter.go index 96f69080f7..af15b0cda7 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -24,10 +24,11 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/registry" ) var ( - registeredExporters = exchange.NewRegistry() + registeredExporters = registry.NewRegistry() ) // Exporter is the interface that all exporters must implement. diff --git a/pkg/mentix/exporters/prometheus/types.go b/pkg/mentix/exchange/exporters/prometheus/types.go similarity index 100% rename from pkg/mentix/exporters/prometheus/types.go rename to pkg/mentix/exchange/exporters/prometheus/types.go diff --git a/pkg/mentix/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go similarity index 99% rename from pkg/mentix/exporters/promsd.go rename to pkg/mentix/exchange/exporters/promsd.go index d1fc753412..ef5fa784da 100755 --- a/pkg/mentix/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -28,7 +28,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters/prometheus" + "github.com/cs3org/reva/pkg/mentix/exchange/exporters/prometheus" "github.com/cs3org/reva/pkg/mentix/meshdata" ) diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exchange/exporters/reqexporter.go similarity index 100% rename from pkg/mentix/exporters/reqexporter.go rename to pkg/mentix/exchange/exporters/reqexporter.go diff --git a/pkg/mentix/exporters/siteloc/query.go b/pkg/mentix/exchange/exporters/siteloc/query.go similarity index 100% rename from pkg/mentix/exporters/siteloc/query.go rename to pkg/mentix/exchange/exporters/siteloc/query.go diff --git a/pkg/mentix/exporters/siteloc/types.go b/pkg/mentix/exchange/exporters/siteloc/types.go similarity index 100% rename from pkg/mentix/exporters/siteloc/types.go rename to pkg/mentix/exchange/exporters/siteloc/types.go diff --git a/pkg/mentix/exporters/sitelocations.go b/pkg/mentix/exchange/exporters/sitelocations.go similarity index 96% rename from pkg/mentix/exporters/sitelocations.go rename to pkg/mentix/exchange/exporters/sitelocations.go index aa88c7c404..cc095b3437 100755 --- a/pkg/mentix/exporters/sitelocations.go +++ b/pkg/mentix/exchange/exporters/sitelocations.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters/siteloc" + "github.com/cs3org/reva/pkg/mentix/exchange/exporters/siteloc" ) // SiteLocationsExporter implements the Site Locations exporter to use with Grafana. diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exchange/exporters/webapi.go similarity index 96% rename from pkg/mentix/exporters/webapi.go rename to pkg/mentix/exchange/exporters/webapi.go index 8a79fb38bc..b055ad17b5 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exchange/exporters/webapi.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exporters/webapi" + "github.com/cs3org/reva/pkg/mentix/exchange/exporters/webapi" ) // WebAPIExporter implements the generic Web API exporter. diff --git a/pkg/mentix/exporters/webapi/query.go b/pkg/mentix/exchange/exporters/webapi/query.go similarity index 100% rename from pkg/mentix/exporters/webapi/query.go rename to pkg/mentix/exchange/exporters/webapi/query.go diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index fb72b926f9..433e98e290 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchange" - "github.com/cs3org/reva/pkg/mentix/exporters" + "github.com/cs3org/reva/pkg/mentix/exchange/exporters" "github.com/cs3org/reva/pkg/mentix/meshdata" ) diff --git a/pkg/mentix/exchange/registry.go b/pkg/mentix/registry/registry.go similarity index 98% rename from pkg/mentix/exchange/registry.go rename to pkg/mentix/registry/registry.go index bf1b763373..bf25127793 100644 --- a/pkg/mentix/exchange/registry.go +++ b/pkg/mentix/registry/registry.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package exchange +package registry import "fmt" From 24eba90dee4298973957aa52c09f92a79f45a17c Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 16:34:20 +0100 Subject: [PATCH 09/35] Linting --- pkg/mentix/connectors/connector.go | 4 ++-- pkg/mentix/exchange/exporters/exporter.go | 4 ++-- pkg/mentix/{ => util}/registry/registry.go | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) rename pkg/mentix/{ => util}/registry/registry.go (72%) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index f293371291..d2b5e8ce3a 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -25,7 +25,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/registry" + "github.com/cs3org/reva/pkg/mentix/util/registry" ) var ( @@ -71,7 +71,7 @@ func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { return nil, err } - var connectors []Connector + connectors := make([]Connector, 0, len(entries)) for _, entry := range entries { connectors = append(connectors, entry.(Connector)) } diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index af15b0cda7..4d2daa8518 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -24,7 +24,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/registry" + "github.com/cs3org/reva/pkg/mentix/util/registry" ) var ( @@ -73,7 +73,7 @@ func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { return nil, err } - var exporters []Exporter + exporters := make([]Exporter, 0, len(entries)) for _, entry := range entries { exporters = append(exporters, entry.(Exporter)) } diff --git a/pkg/mentix/registry/registry.go b/pkg/mentix/util/registry/registry.go similarity index 72% rename from pkg/mentix/registry/registry.go rename to pkg/mentix/util/registry/registry.go index bf25127793..9f362cb6ca 100644 --- a/pkg/mentix/registry/registry.go +++ b/pkg/mentix/util/registry/registry.go @@ -20,19 +20,19 @@ package registry import "fmt" -type RegistryEntry interface { -} - +// Registry represents a simple id->entry map. type Registry struct { - Entries map[string]RegistryEntry + Entries map[string]interface{} } -func (r *Registry) Register(id string, entry RegistryEntry) { +// Register registers a new entry. +func (r *Registry) Register(id string, entry interface{}) { r.Entries[id] = entry } -func (r *Registry) EntriesByID(ids []string) ([]RegistryEntry, error) { - var entries []RegistryEntry +// EntriesByID returns all entries matching the provided IDs. If an entry with a certain ID doesn't exist, an error is returned. +func (r *Registry) EntriesByID(ids []string) ([]interface{}, error) { + var entries []interface{} for _, id := range ids { if entry, ok := r.Entries[id]; ok { entries = append(entries, entry) @@ -46,6 +46,6 @@ func (r *Registry) EntriesByID(ids []string) ([]RegistryEntry, error) { func NewRegistry() *Registry { return &Registry{ - Entries: make(map[string]RegistryEntry), + Entries: make(map[string]interface{}), } } From 65e0e6e28eeb47ddeb15a986bc6240c8e6c48eaa Mon Sep 17 00:00:00 2001 From: Daniel Mueller Date: Thu, 29 Oct 2020 16:51:02 +0100 Subject: [PATCH 10/35] Start work on importers --- internal/http/services/mentix/mentix.go | 1 + pkg/mentix/config/config.go | 1 + pkg/mentix/exchange/importers/importer.go | 59 +++++++++++++++ pkg/mentix/exchange/importers/reqimporter.go | 76 ++++++++++++++++++++ pkg/mentix/mentix.go | 33 +++++++-- 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100755 pkg/mentix/exchange/importers/importer.go create mode 100644 pkg/mentix/exchange/importers/reqimporter.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 8e2f6ec8c7..0a7a7cb7bc 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -104,6 +104,7 @@ func applyInternalConfig(m map[string]interface{}, conf *config.Configuration) { } conf.EnabledConnectors = getSubsections("connectors") + conf.EnabledImporters = getSubsections("importers") conf.EnabledExporters = getSubsections("exporters") } diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index b321cbbcad..61df11cd3a 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -56,6 +56,7 @@ type Configuration struct { // Internal settings EnabledConnectors []string `mapstructure:"-"` + EnabledImporters []string `mapstructure:"-"` EnabledExporters []string `mapstructure:"-"` } diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go new file mode 100755 index 0000000000..1023da882a --- /dev/null +++ b/pkg/mentix/exchange/importers/importer.go @@ -0,0 +1,59 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/util/registry" +) + +var ( + registeredImporters = registry.NewRegistry() +) + +// Importer is the interface that all importers must implement. +type Importer interface { + exchange.Exchanger +} + +// BaseImporter implements basic importer functionality common to all importers. +type BaseImporter struct { + exchange.BaseExchanger +} + +// AvailableImporters returns a list of all importers that are enabled in the configuration. +func AvailableImporters(conf *config.Configuration) ([]Importer, error) { + // Try to add all importers configured in the environment + entries, err := registeredImporters.EntriesByID(conf.EnabledImporters) + if err != nil { + return nil, err + } + + importers := make([]Importer, 0, len(entries)) + for _, entry := range entries { + importers = append(importers, entry.(Importer)) + } + + return importers, nil +} + +func registerImporter(id string, exporter Importer) { + registeredImporters.Register(id, exporter) +} diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchange/importers/reqimporter.go new file mode 100644 index 0000000000..34378c56f9 --- /dev/null +++ b/pkg/mentix/exchange/importers/reqimporter.go @@ -0,0 +1,76 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" +) + +type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) + +// BaseRequestImporter implements basic importer functionality common to all request importers. +type BaseRequestImporter struct { + BaseImporter + exchange.BaseRequestExchanger + + defaultMethodHandler queryCallback +} + +// HandleRequest handles the actual HTTP request. +func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + data, err := importer.handleQuery(req.URL.Query()) + if err == nil { + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } + } else { + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +func (importer *BaseRequestImporter) handleQuery(params url.Values) ([]byte, error) { + // Data is written, so lock it completely + importer.Locker().Lock() + defer importer.Locker().Unlock() + + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + if importer.defaultMethodHandler != nil { + return importer.defaultMethodHandler(importer.MeshData(), params) + } + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } + + return []byte{}, fmt.Errorf("unhandled query for method '%v'", method) +} diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 433e98e290..18e8400cc0 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -31,6 +31,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/exchange/exporters" + "github.com/cs3org/reva/pkg/mentix/exchange/importers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -41,6 +42,7 @@ type Mentix struct { meshData *meshdata.MeshData connectors []connectors.Connector + importers []importers.Importer exporters []exporters.Exporter updateInterval time.Duration @@ -66,6 +68,11 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) return fmt.Errorf("unable to initialize connector: %v", err) } + // Initialize the importers + if err := mntx.initImporters(); err != nil { + return fmt.Errorf("unable to initialize importers: %v", err) + } + // Initialize the exporters if err := mntx.initExporters(); err != nil { return fmt.Errorf("unable to initialize exporters: %v", err) @@ -87,11 +94,15 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) for idx, connector := range mntx.connectors { connectorNames[idx] = connector.GetName() } + importerNames := make([]string, len(mntx.importers)) + for idx, importer := range mntx.importers { + importerNames[idx] = importer.GetName() + } exporterNames := make([]string, len(mntx.exporters)) for idx, exporter := range mntx.exporters { exporterNames[idx] = exporter.GetName() } - log.Info().Msgf("mentix started with connectors: %v; exporters: %v; update interval: %v", strings.Join(connectorNames, ", "), strings.Join(exporterNames, ", "), duration) + log.Info().Msgf("mentix started with connectors: %v; importers: %v; exporters: %v; update interval: %v", strings.Join(connectorNames, ", "), strings.Join(importerNames, ", "), strings.Join(exporterNames, ", "), duration) return nil } @@ -114,6 +125,7 @@ func (mntx *Mentix) initConnectors() error { return nil } +// TODO: init,start,stop -> selbe Fnk. für Importers func (mntx *Mentix) initExporters() error { // Use all exporters exposed by the exporters package exps, err := exporters.AvailableExporters(mntx.conf) @@ -151,7 +163,8 @@ func (mntx *Mentix) stopExporters() { } func (mntx *Mentix) destroy() { - // Stop all exporters + // Stop all im- & exporters + mntx.stopImporters() mntx.stopExporters() } @@ -160,7 +173,10 @@ func (mntx *Mentix) destroy() { func (mntx *Mentix) Run(stopSignal <-chan struct{}) error { defer mntx.destroy() - // Start all exporters; they will be stopped in mntx.destroy + // Start all im- & exporters; they will be stopped in mntx.destroy + if err := mntx.startImporters(); err != nil { + return fmt.Errorf("unable to start importers: %v", err) + } if err := mntx.startExporters(); err != nil { return fmt.Errorf("unable to start exporters: %v", err) } @@ -228,6 +244,7 @@ func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { return nil } +// TODO: selbe Fnk. für Importers // GetRequestExporters returns all exporters that can handle HTTP requests. func (mntx *Mentix) GetRequestExporters() []exchange.RequestExchanger { // Return all exporters that implement the RequestExporter interface @@ -257,6 +274,7 @@ func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { } } +// TODO: handle*request -> selbe Fnk. für Importers func (mntx *Mentix) handleGetRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { // Ask each RequestExporter if it wants to handle the request for _, exporter := range mntx.GetRequestExporters() { @@ -269,7 +287,14 @@ func (mntx *Mentix) handleGetRequest(w http.ResponseWriter, r *http.Request, log } func (mntx *Mentix) handlePostRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { - + // Ask each RequestImporter if it wants to handle the request + for _, importer := range mntx.GetRequestImporters() { + if importer.WantsRequest(r) { + if err := importer.HandleRequest(w, r); err != nil { + log.Err(err).Msg("error handling POST request") + } + } + } } // New creates a new Mentix service instance. From cdf328d9acdd371e4abc34a55820d81c2d1f36d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 30 Oct 2020 12:23:44 +0100 Subject: [PATCH 11/35] Fix example --- examples/mentix/mentix.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index f5d5d950ad..f6fdd36ee2 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -11,7 +11,8 @@ update_interval = "15m" [http.services.mentix.connectors.gocdb] address = "http://sciencemesh-test.uni-muenster.de" -[http.services.mentix.connectors.webapi] +# Enable the WebAPI exporter +[http.services.mentix.exporters.webapi] endpoint = "/" # Enable the CS3API and Site Locations exporters From 7d602f5a80dae341063cc725c381cae16de63d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 30 Oct 2020 13:11:49 +0100 Subject: [PATCH 12/35] Finish work on exchangers --- internal/http/services/mentix/mentix.go | 9 +- pkg/mentix/exchange/exchangers.go | 67 +++++++++++++ pkg/mentix/exchange/exporters/exporter.go | 31 ------ pkg/mentix/exchange/exporters/exporters.go | 89 +++++++++++++++++ pkg/mentix/exchange/importers/importer.go | 26 ----- pkg/mentix/exchange/importers/importers.go | 79 +++++++++++++++ pkg/mentix/mentix.go | 106 +++++++++------------ 7 files changed, 287 insertions(+), 120 deletions(-) create mode 100644 pkg/mentix/exchange/exchangers.go create mode 100644 pkg/mentix/exchange/exporters/exporters.go create mode 100644 pkg/mentix/exchange/importers/importers.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 0a7a7cb7bc..802b7b25bf 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -59,9 +59,14 @@ func (s *svc) Prefix() string { } func (s *svc) Unprotected() []string { - // Get all endpoints exposed by the RequestExporters + // Get all endpoints exposed by the RequestExchangers + importers := s.mntx.GetRequestImporters() exporters := s.mntx.GetRequestExporters() - endpoints := make([]string, len(exporters)) + + endpoints := make([]string, len(importers)+len(exporters)) + for idx, importer := range importers { + endpoints[idx] = importer.Endpoint() + } for idx, exporter := range exporters { endpoints[idx] = exporter.Endpoint() } diff --git a/pkg/mentix/exchange/exchangers.go b/pkg/mentix/exchange/exchangers.go new file mode 100644 index 0000000000..0c2d10ef6d --- /dev/null +++ b/pkg/mentix/exchange/exchangers.go @@ -0,0 +1,67 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exchange + +import ( + "fmt" + + "github.com/cs3org/reva/pkg/mentix/config" + + "github.com/rs/zerolog" +) + +// ActivateExchangers activates the given exchangers. +func ActivateExchangers(exchangers []Exchanger, conf *config.Configuration, log *zerolog.Logger) error { + for _, exchanger := range exchangers { + if err := exchanger.Activate(conf, log); err != nil { + return fmt.Errorf("unable to activate exchanger '%v': %v", exchanger.GetName(), err) + } + } + + return nil +} + +// StartExchangers starts the given exchangers. +func StartExchangers(exchangers []Exchanger) error { + for _, exchanger := range exchangers { + if err := exchanger.Start(); err != nil { + return fmt.Errorf("unable to start exchanger '%v': %v", exchanger.GetName(), err) + } + } + + return nil +} + +// StopExchangers stops the given exchangers. +func StopExchangers(exchangers []Exchanger) { + for _, exchanger := range exchangers { + exchanger.Stop() + } +} + +// GetRequestExchangers gets all exchangers from a vector that implement the RequestExchanger interface. +func GetRequestExchangers(exchangers []Exchanger) []RequestExchanger { + var reqExchangers []RequestExchanger + for _, exporter := range exchangers { + if reqExchanger, ok := exporter.(RequestExchanger); ok { + reqExchangers = append(reqExchangers, reqExchanger) + } + } + return reqExchangers +} diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index 4d2daa8518..4b9fa13661 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -21,14 +21,8 @@ package exporters import ( "fmt" - "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/util/registry" -) - -var ( - registeredExporters = registry.NewRegistry() ) // Exporter is the interface that all exporters must implement. @@ -64,28 +58,3 @@ func (exporter *BaseExporter) storeMeshData(meshData *meshdata.MeshData) error { return nil } - -// AvailableExporters returns a list of all exporters that are enabled in the configuration. -func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { - // Try to add all exporters configured in the environment - entries, err := registeredExporters.EntriesByID(conf.EnabledExporters) - if err != nil { - return nil, err - } - - exporters := make([]Exporter, 0, len(entries)) - for _, entry := range entries { - exporters = append(exporters, entry.(Exporter)) - } - - // At least one exporter must be configured - if len(exporters) == 0 { - return nil, fmt.Errorf("no exporters available") - } - - return exporters, nil -} - -func registerExporter(id string, exporter Exporter) { - registeredExporters.Register(id, exporter) -} diff --git a/pkg/mentix/exchange/exporters/exporters.go b/pkg/mentix/exchange/exporters/exporters.go new file mode 100644 index 0000000000..0a84abe945 --- /dev/null +++ b/pkg/mentix/exchange/exporters/exporters.go @@ -0,0 +1,89 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/util/registry" +) + +// Exporters is a vector of Exporter +type Exporters = []Exporter + +var ( + registeredExporters = registry.NewRegistry() +) + +// AvailableExporters returns a list of all exporters that are enabled in the configuration. +func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { + // Try to add all exporters configured in the environment + entries, err := registeredExporters.EntriesByID(conf.EnabledExporters) + if err != nil { + return nil, err + } + + exporters := make([]Exporter, 0, len(entries)) + for _, entry := range entries { + exporters = append(exporters, entry.(Exporter)) + } + + // At least one exporter must be configured + if len(exporters) == 0 { + return nil, fmt.Errorf("no exporters available") + } + + return exporters, nil +} + +// ActivateExporters activates the given exporters. +func ActivateExporters(exporters []Exporter, conf *config.Configuration, log *zerolog.Logger) error { + return exchange.ActivateExchangers(asExchangers(exporters), conf, log) +} + +// StartExporters starts the given exporters. +func StartExporters(exporters []Exporter) error { + return exchange.StartExchangers(asExchangers(exporters)) +} + +// StopExporters stops the given exporters. +func StopExporters(exporters []Exporter) { + exchange.StopExchangers(asExchangers(exporters)) +} + +func asExchangers(exporters []Exporter) []exchange.Exchanger { + exchangers := make([]exchange.Exchanger, 0, len(exporters)) + for _, exp := range exporters { + exchangers = append(exchangers, exp) + } + return exchangers +} + +// GetRequestExporters returns all exporters that implement the RequestExchanger interface. +func GetRequestExporters(exporters []Exporter) []exchange.RequestExchanger { + return exchange.GetRequestExchangers(asExchangers(exporters)) +} + +func registerExporter(id string, exporter Exporter) { + registeredExporters.Register(id, exporter) +} diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go index 1023da882a..42933d1132 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchange/importers/importer.go @@ -19,13 +19,7 @@ package importers import ( - "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchange" - "github.com/cs3org/reva/pkg/mentix/util/registry" -) - -var ( - registeredImporters = registry.NewRegistry() ) // Importer is the interface that all importers must implement. @@ -37,23 +31,3 @@ type Importer interface { type BaseImporter struct { exchange.BaseExchanger } - -// AvailableImporters returns a list of all importers that are enabled in the configuration. -func AvailableImporters(conf *config.Configuration) ([]Importer, error) { - // Try to add all importers configured in the environment - entries, err := registeredImporters.EntriesByID(conf.EnabledImporters) - if err != nil { - return nil, err - } - - importers := make([]Importer, 0, len(entries)) - for _, entry := range entries { - importers = append(importers, entry.(Importer)) - } - - return importers, nil -} - -func registerImporter(id string, exporter Importer) { - registeredImporters.Register(id, exporter) -} diff --git a/pkg/mentix/exchange/importers/importers.go b/pkg/mentix/exchange/importers/importers.go new file mode 100644 index 0000000000..d76425b345 --- /dev/null +++ b/pkg/mentix/exchange/importers/importers.go @@ -0,0 +1,79 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/util/registry" +) + +var ( + registeredImporters = registry.NewRegistry() +) + +// AvailableImporters returns a list of all importers that are enabled in the configuration. +func AvailableImporters(conf *config.Configuration) ([]Importer, error) { + // Try to add all importers configured in the environment + entries, err := registeredImporters.EntriesByID(conf.EnabledImporters) + if err != nil { + return nil, err + } + + importers := make([]Importer, 0, len(entries)) + for _, entry := range entries { + importers = append(importers, entry.(Importer)) + } + + return importers, nil +} + +// ActivateImporters activates the given importers. +func ActivateImporters(importers []Importer, conf *config.Configuration, log *zerolog.Logger) error { + return exchange.ActivateExchangers(asExchangers(importers), conf, log) +} + +// StartImporters starts the given importers. +func StartImporters(importers []Importer) error { + return exchange.StartExchangers(asExchangers(importers)) +} + +// StopImporters stops the given importers. +func StopImporters(importers []Importer) { + exchange.StopExchangers(asExchangers(importers)) +} + +// GetRequestImporters returns all Importers that implement the RequestExchanger interface. +func GetRequestImporters(importers []Importer) []exchange.RequestExchanger { + return exchange.GetRequestExchangers(asExchangers(importers)) +} + +func asExchangers(importers []Importer) []exchange.Exchanger { + exchangers := make([]exchange.Exchanger, 0, len(importers)) + for _, imp := range importers { + exchangers = append(exchangers, imp) + } + return exchangers +} + +func registerImporter(id string, importer Importer) { + registeredImporters.Register(id, importer) +} diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 18e8400cc0..cf284b0a38 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -68,14 +68,9 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) return fmt.Errorf("unable to initialize connector: %v", err) } - // Initialize the importers - if err := mntx.initImporters(); err != nil { - return fmt.Errorf("unable to initialize importers: %v", err) - } - - // Initialize the exporters - if err := mntx.initExporters(); err != nil { - return fmt.Errorf("unable to initialize exporters: %v", err) + // Initialize the exchangers + if err := mntx.initExchangers(); err != nil { + return fmt.Errorf("unable to initialize exchangers: %v", err) } // Get the update interval @@ -125,47 +120,54 @@ func (mntx *Mentix) initConnectors() error { return nil } -// TODO: init,start,stop -> selbe Fnk. für Importers -func (mntx *Mentix) initExporters() error { +func (mntx *Mentix) initExchangers() error { + // Use all importers exposed by the importers package + imps, err := importers.AvailableImporters(mntx.conf) + if err != nil { + return fmt.Errorf("unable to get registered importers: %v", err) + } + mntx.importers = imps + + if err := importers.ActivateImporters(mntx.importers, mntx.conf, mntx.log); err != nil { + return fmt.Errorf("unable to activate importers: %v", err) + } + // Use all exporters exposed by the exporters package exps, err := exporters.AvailableExporters(mntx.conf) if err != nil { - return fmt.Errorf("unable to get registered exps: %v", err) + return fmt.Errorf("unable to get registered exporters: %v", err) } mntx.exporters = exps - // Activate all exps - for _, exporter := range mntx.exporters { - if err := exporter.Activate(mntx.conf, mntx.log); err != nil { - return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) - } + if err := exporters.ActivateExporters(mntx.exporters, mntx.conf, mntx.log); err != nil { + return fmt.Errorf("unable to activate exporters: %v", err) } return nil } -func (mntx *Mentix) startExporters() error { +func (mntx *Mentix) startExchangers() error { + // Start all importers + if err := importers.StartImporters(mntx.importers); err != nil { + return fmt.Errorf("unable to start importers: %v", err) + } + // Start all exporters - for _, exporter := range mntx.exporters { - if err := exporter.Start(); err != nil { - return fmt.Errorf("unable to start exporter '%v': %v", exporter.GetName(), err) - } + if err := exporters.StartExporters(mntx.exporters); err != nil { + return fmt.Errorf("unable to start exporters: %v", err) } return nil } -func (mntx *Mentix) stopExporters() { - // Stop all exporters - for _, exporter := range mntx.exporters { - exporter.Stop() - } +func (mntx *Mentix) stopExchangers() { + exporters.StopExporters(mntx.exporters) + importers.StopImporters(mntx.importers) } func (mntx *Mentix) destroy() { // Stop all im- & exporters - mntx.stopImporters() - mntx.stopExporters() + mntx.stopExchangers() } // Run starts the Mentix service that will periodically pull the configured data source and publish this data @@ -174,11 +176,8 @@ func (mntx *Mentix) Run(stopSignal <-chan struct{}) error { defer mntx.destroy() // Start all im- & exporters; they will be stopped in mntx.destroy - if err := mntx.startImporters(); err != nil { - return fmt.Errorf("unable to start importers: %v", err) - } - if err := mntx.startExporters(); err != nil { - return fmt.Errorf("unable to start exporters: %v", err) + if err := mntx.startExchangers(); err != nil { + return fmt.Errorf("unable to start exchangers: %v", err) } updateTimestamp := time.Time{} @@ -244,54 +243,39 @@ func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { return nil } -// TODO: selbe Fnk. für Importers +// GetRequestImporters returns all exporters that can handle HTTP requests. +func (mntx *Mentix) GetRequestImporters() []exchange.RequestExchanger { + return importers.GetRequestImporters(mntx.importers) +} + // GetRequestExporters returns all exporters that can handle HTTP requests. func (mntx *Mentix) GetRequestExporters() []exchange.RequestExchanger { - // Return all exporters that implement the RequestExporter interface - var reqExporters []exchange.RequestExchanger - for _, exporter := range mntx.exporters { - if reqExporter, ok := exporter.(exchange.RequestExchanger); ok { - reqExporters = append(reqExporters, reqExporter) - } - } - return reqExporters + return exporters.GetRequestExporters(mntx.exporters) } -// RequestHandler handles any incoming HTTP requests by asking each RequestExporter whether it wants to +// RequestHandler handles any incoming HTTP requests by asking each RequestExchanger whether it wants to // handle the request (usually based on the relative URL path). func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { log := appctx.GetLogger(r.Context()) switch r.Method { case http.MethodGet: - mntx.handleGetRequest(w, r, log) + mntx.handleRequest(mntx.GetRequestExporters(), w, r, log) case http.MethodPost: - mntx.handlePostRequest(w, r, log) + mntx.handleRequest(mntx.GetRequestImporters(), w, r, log) default: log.Err(fmt.Errorf("unsupported method")).Msg("error handling incoming request") } } -// TODO: handle*request -> selbe Fnk. für Importers -func (mntx *Mentix) handleGetRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { - // Ask each RequestExporter if it wants to handle the request - for _, exporter := range mntx.GetRequestExporters() { +func (mntx *Mentix) handleRequest(exchangers []exchange.RequestExchanger, w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { + // Ask each RequestExchanger if it wants to handle the request + for _, exporter := range exchangers { if exporter.WantsRequest(r) { if err := exporter.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling GET request") - } - } - } -} - -func (mntx *Mentix) handlePostRequest(w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { - // Ask each RequestImporter if it wants to handle the request - for _, importer := range mntx.GetRequestImporters() { - if importer.WantsRequest(r) { - if err := importer.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling POST request") + log.Err(err).Msg("error handling request") } } } From 6939f97b885af4cd0e52f7b0be8a179a6549f670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 2 Nov 2020 12:49:29 +0100 Subject: [PATCH 13/35] Change how mesh data is stored in the Mentix core --- pkg/mentix/connectors/connector.go | 22 ++++++++++++-- pkg/mentix/connectors/gocdb.go | 4 ++- pkg/mentix/connectors/localfile.go | 4 ++- pkg/mentix/exchange/exchanger.go | 6 ++-- pkg/mentix/exchange/exporters/exporter.go | 29 ++++++++++-------- pkg/mentix/exchange/exporters/promsd.go | 6 ++-- pkg/mentix/mentix.go | 36 +++++++++++++++-------- pkg/mentix/meshdata/meshdata.go | 4 +++ pkg/mentix/meshdata/meshdataset.go | 31 +++++++++++++++++++ 9 files changed, 107 insertions(+), 35 deletions(-) create mode 100644 pkg/mentix/meshdata/meshdataset.go diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index d2b5e8ce3a..716d86d200 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -34,6 +34,9 @@ var ( // Connector is the interface that all connectors must implement. type Connector interface { + // GetID returns the ID of the connector. + GetID() string + // Activate activates a connector. Activate(conf *config.Configuration, log *zerolog.Logger) error // RetrieveMeshData fetches new mesh data. @@ -45,10 +48,25 @@ type Connector interface { // BaseConnector implements basic connector functionality common to all connectors. type BaseConnector struct { + id string + conf *config.Configuration log *zerolog.Logger } +// GetID returns the ID of the connector. +func (connector *BaseConnector) GetID() string { + return connector.id +} + +// SetID sets the ID of the connector. +func (connector *BaseConnector) SetID(id string) { + // The ID can only be set once + if connector.id == "" { + connector.id = id + } +} + // Activate activates the connector. func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { @@ -84,6 +102,6 @@ func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { return connectors, nil } -func registerConnector(id string, connector Connector) { - registeredConnectors.Register(id, connector) +func registerConnector(connector Connector) { + registeredConnectors.Register(connector.GetID(), connector) } diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 07daceed69..2663d1933d 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -264,5 +264,7 @@ func (connector *GOCDBConnector) GetName() string { } func init() { - registerConnector(config.ConnectorIDGOCDB, &GOCDBConnector{}) + connector := &GOCDBConnector{} + connector.SetID(config.ConnectorIDGOCDB) + registerConnector(connector) } diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 8323f10b0f..bb5cb916e2 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -89,5 +89,7 @@ func (connector *LocalFileConnector) GetName() string { } func init() { - registerConnector(config.ConnectorIDLocalFile, &LocalFileConnector{}) + connector := &LocalFileConnector{} + connector.SetID(config.ConnectorIDLocalFile) + registerConnector(connector) } diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchange/exchanger.go index ed550b674c..69379b601f 100644 --- a/pkg/mentix/exchange/exchanger.go +++ b/pkg/mentix/exchange/exchanger.go @@ -43,7 +43,7 @@ type Exchanger interface { // Stop stops any running background activities of the exchanger. Stop() - // MeshData returns the mesh data object. + // MeshDataSet returns the mesh data. MeshData() *meshdata.MeshData // GetName returns the display name of the exchanger. @@ -95,12 +95,12 @@ func (exchanger *BaseExchanger) Log() *zerolog.Logger { return exchanger.log } -// MeshData returns the mesh data object. +// MeshDataSet returns the stored mesh data. func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { return exchanger.meshData } -// SetMeshData sets new mesh data. +// SetMeshDataSet sets new mesh data. func (exchanger *BaseExchanger) SetMeshData(meshData *meshdata.MeshData) { exchanger.Locker().Lock() defer exchanger.Locker().Unlock() diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index 4b9fa13661..bb7dc5d011 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -29,8 +29,8 @@ import ( type Exporter interface { exchange.Exchanger - // UpdateMeshData is called whenever the mesh data has changed to reflect these changes. - UpdateMeshData(*meshdata.MeshData) error + // UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. + UpdateMeshDataSet(meshdata.MeshDataSet) error } // BaseExporter implements basic exporter functionality common to all exporters. @@ -38,23 +38,28 @@ type BaseExporter struct { exchange.BaseExchanger } -// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. -func (exporter *BaseExporter) UpdateMeshData(meshData *meshdata.MeshData) error { - // Update the stored mesh data - if err := exporter.storeMeshData(meshData); err != nil { +// UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. +func (exporter *BaseExporter) UpdateMeshDataSet(meshDataSet meshdata.MeshDataSet) error { + // Update the stored mesh data set + if err := exporter.storeMeshDataSet(meshDataSet); err != nil { return fmt.Errorf("unable to store the mesh data: %v", err) } return nil } -func (exporter *BaseExporter) storeMeshData(meshData *meshdata.MeshData) error { - // Store the new mesh data by cloning it - meshDataCloned := meshData.Clone() - if meshDataCloned == nil { - return fmt.Errorf("unable to clone the mesh data") +func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.MeshDataSet) error { + // TODO: Filter based on connectorID + // Store the new mesh data set by cloning it and then merging the cloned data into one object + meshDataSetCloned := make(meshdata.MeshDataSet) + for connectorID, meshData := range meshDataSet { + meshDataCloned := meshData.Clone() + if meshDataCloned == nil { + return fmt.Errorf("unable to clone the mesh data") + } + meshDataSetCloned[connectorID] = meshDataCloned } - exporter.SetMeshData(meshDataCloned) + exporter.SetMeshData(meshdata.MergeMeshDataSet(meshDataSetCloned)) return nil } diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index ef5fa784da..8a7af79825 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -133,9 +133,9 @@ func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log * return nil } -// UpdateMeshData is called whenever the mesh data has changed to reflect these changes. -func (exporter *PrometheusSDExporter) UpdateMeshData(meshData *meshdata.MeshData) error { - if err := exporter.BaseExporter.UpdateMeshData(meshData); err != nil { +// UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. +func (exporter *PrometheusSDExporter) UpdateMeshDataSet(meshDataSet meshdata.MeshDataSet) error { + if err := exporter.BaseExporter.UpdateMeshDataSet(meshDataSet); err != nil { return err } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index cf284b0a38..616fd3255a 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -40,11 +40,12 @@ type Mentix struct { conf *config.Configuration log *zerolog.Logger - meshData *meshdata.MeshData connectors []connectors.Connector importers []importers.Importer exporters []exporters.Exporter + meshDataSet meshdata.MeshDataSet + updateInterval time.Duration } @@ -81,8 +82,8 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) } mntx.updateInterval = duration - // Create empty mesh data - mntx.meshData = meshdata.New() + // Create empty mesh data set + mntx.meshDataSet = make(meshdata.MeshDataSet) // Log some infos connectorNames := make([]string, len(mntx.connectors)) @@ -195,9 +196,9 @@ loop: // If enough time has passed, retrieve the latest mesh data and update it if time.Since(updateTimestamp) >= mntx.updateInterval { - meshData, err := mntx.retrieveMeshData() + meshDataSet, err := mntx.retrieveMeshDataSet() if err == nil { - if err := mntx.applyMeshData(meshData); err != nil { + if err := mntx.applyMeshDataSet(meshDataSet); err != nil { mntx.log.Err(err).Msg("failed to apply mesh data") } } else { @@ -213,28 +214,37 @@ loop: return nil } -func (mntx *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { - var mergedMeshData meshdata.MeshData +func (mntx *Mentix) retrieveMeshDataSet() (meshdata.MeshDataSet, error) { + meshDataSet := make(meshdata.MeshDataSet) for _, connector := range mntx.connectors { meshData, err := connector.RetrieveMeshData() if err != nil { return nil, fmt.Errorf("retrieving mesh data from connector '%v' failed: %v", connector.GetName(), err) } - mergedMeshData.Merge(meshData) + meshDataSet[connector.GetID()] = meshData } - return &mergedMeshData, nil + return meshDataSet, nil } -func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { - if !meshData.Compare(mntx.meshData) { +func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.MeshDataSet) error { + // Check if mesh data from any connector has changed + meshDataChanged := false + for connectorID, meshData := range meshDataSet { + if !meshData.Compare(mntx.meshDataSet[connectorID]) { + meshDataChanged = true + break + } + } + + if meshDataChanged { mntx.log.Debug().Msg("mesh data changed, applying") - mntx.meshData = meshData + mntx.meshDataSet = meshDataSet for _, exporter := range mntx.exporters { - if err := exporter.UpdateMeshData(meshData); err != nil { + if err := exporter.UpdateMeshDataSet(meshDataSet); err != nil { return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) } } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 46d4aa1179..d2935b1e64 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -77,6 +77,10 @@ func (meshData *MeshData) Clone() *MeshData { // Compare checks whether the stored data equals the data of another MeshData object. func (meshData *MeshData) Compare(other *MeshData) bool { + if other == nil { + return false + } + // To avoid cumbersome comparisons, just compare the JSON-encoded data json1, _ := meshData.ToJSON() json2, _ := other.ToJSON() diff --git a/pkg/mentix/meshdata/meshdataset.go b/pkg/mentix/meshdata/meshdataset.go new file mode 100644 index 0000000000..81e16445f4 --- /dev/null +++ b/pkg/mentix/meshdata/meshdataset.go @@ -0,0 +1,31 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +// MeshDataSet represents a set of MeshData objects. +type MeshDataSet = map[string]*MeshData + +// MergeMeshDataSet merges all mesh data objects within a set. +func MergeMeshDataSet(meshDataSet MeshDataSet) *MeshData { + mergedMeshData := &MeshData{} + for _, meshData := range meshDataSet { + mergedMeshData.Merge(meshData) + } + return mergedMeshData +} From 53c4e92d011fac843aa9f9a8242f937e28502529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 2 Nov 2020 13:34:40 +0100 Subject: [PATCH 14/35] Ability to limit exchangers to specific connectors --- internal/http/services/mentix/mentix.go | 9 ++++++++ pkg/mentix/config/config.go | 14 +++++++---- pkg/mentix/exchange/exchanger.go | 23 +++++++++++++++++++ pkg/mentix/exchange/exporters/cs3api.go | 2 ++ pkg/mentix/exchange/exporters/exporter.go | 5 +++- pkg/mentix/exchange/exporters/promsd.go | 3 +++ .../exchange/exporters/sitelocations.go | 2 ++ pkg/mentix/exchange/exporters/webapi.go | 2 ++ 8 files changed, 54 insertions(+), 6 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 802b7b25bf..fbb73797a5 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -129,14 +129,23 @@ func applyDefaultConfig(conf *config.Configuration) { if conf.Exporters.WebAPI.Endpoint == "" { conf.Exporters.WebAPI.Endpoint = "/" } + if len(conf.Exporters.WebAPI.EnabledConnectors) == 0 { + conf.Exporters.WebAPI.EnabledConnectors = append(conf.Exporters.WebAPI.EnabledConnectors, "*") + } if conf.Exporters.CS3API.Endpoint == "" { conf.Exporters.CS3API.Endpoint = "/cs3" } + if len(conf.Exporters.CS3API.EnabledConnectors) == 0 { + conf.Exporters.CS3API.EnabledConnectors = append(conf.Exporters.CS3API.EnabledConnectors, "*") + } if conf.Exporters.SiteLocations.Endpoint == "" { conf.Exporters.SiteLocations.Endpoint = "/loc" } + if len(conf.Exporters.SiteLocations.EnabledConnectors) == 0 { + conf.Exporters.SiteLocations.EnabledConnectors = append(conf.Exporters.SiteLocations.EnabledConnectors, "*") + } } // New returns a new Mentix service. diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 61df11cd3a..3673c09f68 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -37,20 +37,24 @@ type Configuration struct { Exporters struct { WebAPI struct { - Endpoint string `mapstructure:"endpoint"` + Endpoint string `mapstructure:"endpoint"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` CS3API struct { - Endpoint string `mapstructure:"endpoint"` + Endpoint string `mapstructure:"endpoint"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"cs3api"` SiteLocations struct { - Endpoint string `mapstructure:"endpoint"` + Endpoint string `mapstructure:"endpoint"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"siteloc"` PrometheusSD struct { - MetricsOutputFile string `mapstructure:"metrics_output_file"` - BlackboxOutputFile string `mapstructure:"blackbox_output_file"` + MetricsOutputFile string `mapstructure:"metrics_output_file"` + BlackboxOutputFile string `mapstructure:"blackbox_output_file"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"promsd"` } `mapstructure:"exporters"` diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchange/exchanger.go index 69379b601f..0ce0422b35 100644 --- a/pkg/mentix/exchange/exchanger.go +++ b/pkg/mentix/exchange/exchanger.go @@ -26,6 +26,7 @@ package exchange import ( "fmt" + "strings" "sync" "github.com/rs/zerolog" @@ -57,6 +58,8 @@ type BaseExchanger struct { conf *config.Configuration log *zerolog.Logger + enabledConnectors []string + meshData *meshdata.MeshData locker sync.RWMutex } @@ -85,6 +88,16 @@ func (exchanger *BaseExchanger) Start() error { func (exchanger *BaseExchanger) Stop() { } +// IsConnectorEnabled checks if the given connector is enabled for the exchanger. +func (exchanger *BaseExchanger) IsConnectorEnabled(id string) bool { + for _, connectorID := range exchanger.enabledConnectors { + if connectorID == "*" || strings.EqualFold(connectorID, id) { + return true + } + } + return false +} + // Config returns the configuration object. func (exchanger *BaseExchanger) Config() *config.Configuration { return exchanger.conf @@ -95,6 +108,16 @@ func (exchanger *BaseExchanger) Log() *zerolog.Logger { return exchanger.log } +// EnabledConnectors returns the list of all enabled connectors for the exchanger. +func (exchanger *BaseExchanger) EnabledConnectors() []string { + return exchanger.enabledConnectors +} + +// SetEnabledConnectors sets the list of all enabled connectors for the exchanger. +func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) { + exchanger.enabledConnectors = connectors +} + // MeshDataSet returns the stored mesh data. func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { return exchanger.meshData diff --git a/pkg/mentix/exchange/exporters/cs3api.go b/pkg/mentix/exchange/exporters/cs3api.go index 3bbcca340f..b3e5247e30 100755 --- a/pkg/mentix/exchange/exporters/cs3api.go +++ b/pkg/mentix/exchange/exporters/cs3api.go @@ -38,6 +38,8 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo // Store CS3API specifics exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint) + exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors) + exporter.defaultMethodHandler = cs3api.HandleDefaultQuery return nil diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index bb7dc5d011..c19302166a 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -49,10 +49,13 @@ func (exporter *BaseExporter) UpdateMeshDataSet(meshDataSet meshdata.MeshDataSet } func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.MeshDataSet) error { - // TODO: Filter based on connectorID // Store the new mesh data set by cloning it and then merging the cloned data into one object meshDataSetCloned := make(meshdata.MeshDataSet) for connectorID, meshData := range meshDataSet { + if !exporter.IsConnectorEnabled(connectorID) { + continue + } + meshDataCloned := meshData.Clone() if meshDataCloned == nil { return fmt.Errorf("unable to clone the mesh data") diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index 8a7af79825..15d944b59d 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -130,6 +130,9 @@ func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log * } } + // Store PrometheusSD specifics + exporter.SetEnabledConnectors(conf.Exporters.PrometheusSD.EnabledConnectors) + return nil } diff --git a/pkg/mentix/exchange/exporters/sitelocations.go b/pkg/mentix/exchange/exporters/sitelocations.go index cc095b3437..21190f8a12 100755 --- a/pkg/mentix/exchange/exporters/sitelocations.go +++ b/pkg/mentix/exchange/exporters/sitelocations.go @@ -38,6 +38,8 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log // Store SiteLocations specifics exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint) + exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors) + exporter.defaultMethodHandler = siteloc.HandleDefaultQuery return nil diff --git a/pkg/mentix/exchange/exporters/webapi.go b/pkg/mentix/exchange/exporters/webapi.go index b055ad17b5..764a354323 100755 --- a/pkg/mentix/exchange/exporters/webapi.go +++ b/pkg/mentix/exchange/exporters/webapi.go @@ -38,6 +38,8 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo // Store WebAPI specifics exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint) + exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) + exporter.defaultMethodHandler = webapi.HandleDefaultQuery return nil From 04679adf7800abb6131f21edae4872df014dcce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 2 Nov 2020 13:44:09 +0100 Subject: [PATCH 15/35] Update example and docs --- .../en/docs/config/http/services/mentix/cs3api/_index.md | 8 ++++++++ .../en/docs/config/http/services/mentix/promsd/_index.md | 8 ++++++++ .../en/docs/config/http/services/mentix/siteloc/_index.md | 8 ++++++++ .../en/docs/config/http/services/mentix/webapi/_index.md | 8 ++++++++ examples/mentix/mentix.toml | 6 ++++++ 5 files changed, 38 insertions(+) diff --git a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md index da86082dee..d2707bae1c 100644 --- a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md @@ -17,3 +17,11 @@ The endpoint where the mesh data can be queried. endpoint = "/data" {{< /highlight >}} {{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="*" %}} +A list of all enabled connectors for the exporter. +{{< highlight toml >}} +[http.services.mentix.exporters.cs3api] +enabled_connectors = ["gocdb"] +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md index 0b69756af3..20cb66952a 100644 --- a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md @@ -25,3 +25,11 @@ The target filename of the generated Prometheus File SD scrape config for the bl blackbox_output_file = "/var/shared/prometheus/blackbox.json" {{< /highlight >}} {{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="*" %}} +A list of all enabled connectors for the exporter. +{{< highlight toml >}} +[http.services.mentix.exporters.promsd] +enabled_connectors = ["gocdb"] +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md index 5ccc48c613..d33477dbc2 100644 --- a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -17,3 +17,11 @@ The endpoint where the locations data can be queried. endpoint = "/loc" {{< /highlight >}} {{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="*" %}} +A list of all enabled connectors for the exporter. +{{< highlight toml >}} +[http.services.mentix.exporters.siteloc] +enabled_connectors = ["gocdb"] +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index edfaa75668..af61e94673 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -17,3 +17,11 @@ The endpoint where the mesh data can be queried. endpoint = "/data" {{< /highlight >}} {{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="*" %}} +A list of all enabled connectors for the exporter. +{{< highlight toml >}} +[http.services.mentix.exporters.webapi] +enabled_connectors = ["gocdb"] +{{< /highlight >}} +{{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index f6fdd36ee2..dfd5359fb2 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -11,6 +11,10 @@ update_interval = "15m" [http.services.mentix.connectors.gocdb] address = "http://sciencemesh-test.uni-muenster.de" +# Sites can also be stored in a local file +[http.services.mentix.connectors.localfile] +file = "/usr/share/revad/sites.json" + # Enable the WebAPI exporter [http.services.mentix.exporters.webapi] endpoint = "/" @@ -18,6 +22,8 @@ endpoint = "/" # Enable the CS3API and Site Locations exporters [http.services.mentix.exporters.cs3api] [http.services.mentix.exporters.siteloc] +# If this setting is omitted, all connectors will be used as data sources +enabled_connectors = ["gocdb"] # Configure the Prometheus Service Discovery: # [http.services.mentix.exporters.promsd] From 87140354a2368f81dd7de253a050e8b7c2a4725a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 2 Nov 2020 14:37:36 +0100 Subject: [PATCH 16/35] Ground work for importers --- pkg/mentix/exchange/exporters/cs3api.go | 2 +- pkg/mentix/exchange/exporters/cs3api/query.go | 9 ++-- pkg/mentix/exchange/exporters/exporter.go | 8 ++-- pkg/mentix/exchange/exporters/promsd.go | 6 +-- pkg/mentix/exchange/exporters/reqexporter.go | 23 ++++++----- .../exchange/exporters/siteloc/query.go | 9 ++-- .../exchange/exporters/sitelocations.go | 2 +- pkg/mentix/exchange/exporters/webapi.go | 2 +- pkg/mentix/exchange/exporters/webapi/query.go | 7 ++-- pkg/mentix/exchange/importers/importer.go | 8 ++++ pkg/mentix/exchange/importers/reqimporter.go | 41 ++++++++++++++----- pkg/mentix/mentix.go | 21 +++++++++- 12 files changed, 93 insertions(+), 45 deletions(-) diff --git a/pkg/mentix/exchange/exporters/cs3api.go b/pkg/mentix/exchange/exporters/cs3api.go index b3e5247e30..d3674f0dfc 100755 --- a/pkg/mentix/exchange/exporters/cs3api.go +++ b/pkg/mentix/exchange/exporters/cs3api.go @@ -40,7 +40,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint) exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors) - exporter.defaultMethodHandler = cs3api.HandleDefaultQuery + exporter.defaultActionHandler = cs3api.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exchange/exporters/cs3api/query.go b/pkg/mentix/exchange/exporters/cs3api/query.go index 53e016e28b..7419928105 100755 --- a/pkg/mentix/exchange/exporters/cs3api/query.go +++ b/pkg/mentix/exchange/exporters/cs3api/query.go @@ -21,6 +21,7 @@ package cs3api import ( "encoding/json" "fmt" + "net/http" "net/url" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" @@ -29,20 +30,20 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { - return []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) } // Marshal the OCM data as JSON data, err := json.MarshalIndent(ocmData, "", "\t") if err != nil { - return []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) } - return data, nil + return http.StatusOK, data, nil } func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) { diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index c19302166a..b8bb44520c 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -29,8 +29,8 @@ import ( type Exporter interface { exchange.Exchanger - // UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. - UpdateMeshDataSet(meshdata.MeshDataSet) error + // Update is called whenever the mesh data set has changed to reflect these changes. + Update(meshdata.MeshDataSet) error } // BaseExporter implements basic exporter functionality common to all exporters. @@ -38,8 +38,8 @@ type BaseExporter struct { exchange.BaseExchanger } -// UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. -func (exporter *BaseExporter) UpdateMeshDataSet(meshDataSet meshdata.MeshDataSet) error { +// Update is called whenever the mesh data set has changed to reflect these changes. +func (exporter *BaseExporter) Update(meshDataSet meshdata.MeshDataSet) error { // Update the stored mesh data set if err := exporter.storeMeshDataSet(meshDataSet); err != nil { return fmt.Errorf("unable to store the mesh data: %v", err) diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index 15d944b59d..aa1954a0ce 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -136,9 +136,9 @@ func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log * return nil } -// UpdateMeshDataSet is called whenever the mesh data set has changed to reflect these changes. -func (exporter *PrometheusSDExporter) UpdateMeshDataSet(meshDataSet meshdata.MeshDataSet) error { - if err := exporter.BaseExporter.UpdateMeshDataSet(meshDataSet); err != nil { +// Update is called whenever the mesh data set has changed to reflect these changes. +func (exporter *PrometheusSDExporter) Update(meshDataSet meshdata.MeshDataSet) error { + if err := exporter.BaseExporter.Update(meshDataSet); err != nil { return err } diff --git a/pkg/mentix/exchange/exporters/reqexporter.go b/pkg/mentix/exchange/exporters/reqexporter.go index a4804bf205..35e7628fce 100644 --- a/pkg/mentix/exchange/exporters/reqexporter.go +++ b/pkg/mentix/exchange/exporters/reqexporter.go @@ -29,23 +29,24 @@ import ( ) const ( - queryMethodDefault = "" + queryActionDefault = "" ) -type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) +type queryCallback func(*meshdata.MeshData, url.Values) (int, []byte, error) // BaseRequestExporter implements basic exporter functionality common to all request exporters. type BaseRequestExporter struct { BaseExporter exchange.BaseRequestExchanger - defaultMethodHandler queryCallback + defaultActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - data, err := exporter.handleQuery(req.URL.Query()) + status, data, err := exporter.handleQuery(req.URL.Query()) if err == nil { + resp.WriteHeader(status) if _, err := resp.Write(data); err != nil { return fmt.Errorf("error writing the API request response: %v", err) } @@ -56,21 +57,21 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req return nil } -func (exporter *BaseRequestExporter) handleQuery(params url.Values) ([]byte, error) { +func (exporter *BaseRequestExporter) handleQuery(params url.Values) (int, []byte, error) { // Data is read, so lock it for writing exporter.Locker().RLock() defer exporter.Locker().RUnlock() - method := params.Get("method") + method := params.Get("action") switch strings.ToLower(method) { - case queryMethodDefault: - if exporter.defaultMethodHandler != nil { - return exporter.defaultMethodHandler(exporter.MeshData(), params) + case queryActionDefault: + if exporter.defaultActionHandler != nil { + return exporter.defaultActionHandler(exporter.MeshData(), params) } default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) + return http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", method) } - return []byte{}, fmt.Errorf("unhandled query for method '%v'", method) + return http.StatusNotImplemented, []byte{}, fmt.Errorf("unhandled query for action '%v'", method) } diff --git a/pkg/mentix/exchange/exporters/siteloc/query.go b/pkg/mentix/exchange/exporters/siteloc/query.go index a4907ac66a..ad7434b939 100755 --- a/pkg/mentix/exchange/exporters/siteloc/query.go +++ b/pkg/mentix/exchange/exporters/siteloc/query.go @@ -21,26 +21,27 @@ package siteloc import ( "encoding/json" "fmt" + "net/http" "net/url" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { // Convert the mesh data locData, err := convertMeshDataToLocationData(meshData) if err != nil { - return []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) } // Marshal the location data as JSON data, err := json.MarshalIndent(locData, "", "\t") if err != nil { - return []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) } - return data, nil + return http.StatusOK, data, nil } func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { diff --git a/pkg/mentix/exchange/exporters/sitelocations.go b/pkg/mentix/exchange/exporters/sitelocations.go index 21190f8a12..21f31cdc7e 100755 --- a/pkg/mentix/exchange/exporters/sitelocations.go +++ b/pkg/mentix/exchange/exporters/sitelocations.go @@ -40,7 +40,7 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint) exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors) - exporter.defaultMethodHandler = siteloc.HandleDefaultQuery + exporter.defaultActionHandler = siteloc.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exchange/exporters/webapi.go b/pkg/mentix/exchange/exporters/webapi.go index 764a354323..e981c15456 100755 --- a/pkg/mentix/exchange/exporters/webapi.go +++ b/pkg/mentix/exchange/exporters/webapi.go @@ -40,7 +40,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint) exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) - exporter.defaultMethodHandler = webapi.HandleDefaultQuery + exporter.defaultActionHandler = webapi.HandleDefaultQuery return nil } diff --git a/pkg/mentix/exchange/exporters/webapi/query.go b/pkg/mentix/exchange/exporters/webapi/query.go index e413b8375c..3e2027fa8e 100755 --- a/pkg/mentix/exchange/exporters/webapi/query.go +++ b/pkg/mentix/exchange/exporters/webapi/query.go @@ -21,18 +21,19 @@ package webapi import ( "encoding/json" "fmt" + "net/http" "net/url" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { - return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) } - return data, nil + return http.StatusOK, data, nil } diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go index 42933d1132..21d433ae2a 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchange/importers/importer.go @@ -25,9 +25,17 @@ import ( // Importer is the interface that all importers must implement. type Importer interface { exchange.Exchanger + + // Process is called periodically to perform its tasks. + Process() error } // BaseImporter implements basic importer functionality common to all importers. type BaseImporter struct { exchange.BaseExchanger } + +// Process is called periodically to perform its tasks. +func (importer *BaseImporter) Process() error { + return nil +} diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchange/importers/reqimporter.go index 34378c56f9..61a965aa36 100644 --- a/pkg/mentix/exchange/importers/reqimporter.go +++ b/pkg/mentix/exchange/importers/reqimporter.go @@ -29,23 +29,28 @@ import ( ) const ( - queryMethodDefault = "" + queryActionRegisterSite = "register" + queryActionUnregisterSite = "unregister" ) -type queryCallback func(*meshdata.MeshData, url.Values) ([]byte, error) +type queryCallback func(url.Values) (*meshdata.MeshData, int, []byte, error) // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { BaseImporter exchange.BaseRequestExchanger - defaultMethodHandler queryCallback + registerSiteActionHandler queryCallback + unregisterSiteActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - data, err := importer.handleQuery(req.URL.Query()) + meshData, status, data, err := importer.handleQuery(req.URL.Query()) if err == nil { + importer.mergeImportedMeshData(meshData) + + resp.WriteHeader(status) if _, err := resp.Write(data); err != nil { return fmt.Errorf("error writing the API request response: %v", err) } @@ -56,21 +61,35 @@ func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req return nil } -func (importer *BaseRequestImporter) handleQuery(params url.Values) ([]byte, error) { +func (importer *BaseRequestImporter) mergeImportedMeshData(meshData *meshdata.MeshData) { // Data is written, so lock it completely importer.Locker().Lock() defer importer.Locker().Unlock() - method := params.Get("method") + // Merge the newly imported data with any existing data stored in the importer + if meshDataOld := importer.MeshData(); meshDataOld != nil { + meshDataOld.Merge(meshData) + } else { + importer.SetMeshData(meshData) + } +} + +func (importer *BaseRequestImporter) handleQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { + method := params.Get("action") switch strings.ToLower(method) { - case queryMethodDefault: - if importer.defaultMethodHandler != nil { - return importer.defaultMethodHandler(importer.MeshData(), params) + case queryActionRegisterSite: + if importer.registerSiteActionHandler != nil { + return importer.registerSiteActionHandler(params) + } + + case queryActionUnregisterSite: + if importer.unregisterSiteActionHandler != nil { + return importer.unregisterSiteActionHandler(params) } default: - return []byte{}, fmt.Errorf("unknown API method '%v'", method) + return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", method) } - return []byte{}, fmt.Errorf("unhandled query for method '%v'", method) + return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", method) } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 616fd3255a..23c8a57830 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -194,8 +194,15 @@ loop: } } - // If enough time has passed, retrieve the latest mesh data and update it + // If enough time has passed, process im- and exports of mesh data if time.Since(updateTimestamp) >= mntx.updateInterval { + // Let all importers do their work first + if err := mntx.processImporters(); err != nil { + mntx.log.Err(err).Msgf("an error occurred while processing the importers: %v", err) + } + + // Retrieve and update the mesh data; if the importers modified any data, these changes will + // be reflected automatically here meshDataSet, err := mntx.retrieveMeshDataSet() if err == nil { if err := mntx.applyMeshDataSet(meshDataSet); err != nil { @@ -214,6 +221,16 @@ loop: return nil } +func (mntx *Mentix) processImporters() error { + for _, importer := range mntx.importers { + if err := importer.Process(); err != nil { + return fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) + } + } + + return nil +} + func (mntx *Mentix) retrieveMeshDataSet() (meshdata.MeshDataSet, error) { meshDataSet := make(meshdata.MeshDataSet) @@ -244,7 +261,7 @@ func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.MeshDataSet) error { mntx.meshDataSet = meshDataSet for _, exporter := range mntx.exporters { - if err := exporter.UpdateMeshDataSet(meshDataSet); err != nil { + if err := exporter.Update(meshDataSet); err != nil { return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) } } From 54bf0a1c6a7b838d462009f816b804a46f44cbe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 2 Nov 2020 16:22:48 +0100 Subject: [PATCH 17/35] Correct handling of queries --- internal/http/services/mentix/mentix.go | 26 ++++++--- pkg/mentix/config/config.go | 7 +++ pkg/mentix/config/ids.go | 3 + pkg/mentix/exchange/exporters/reqexporter.go | 25 ++++----- pkg/mentix/exchange/importers/reqimporter.go | 33 +++++------ pkg/mentix/exchange/importers/webapi.go | 56 +++++++++++++++++++ pkg/mentix/exchange/importers/webapi/query.go | 38 +++++++++++++ pkg/mentix/exchange/reqexchanger.go | 2 +- pkg/mentix/mentix.go | 8 +-- 9 files changed, 149 insertions(+), 49 deletions(-) create mode 100755 pkg/mentix/exchange/importers/webapi.go create mode 100755 pkg/mentix/exchange/importers/webapi/query.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index fbb73797a5..8c7fff572e 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -114,6 +114,7 @@ func applyInternalConfig(m map[string]interface{}, conf *config.Configuration) { } func applyDefaultConfig(conf *config.Configuration) { + // General if conf.Prefix == "" { conf.Prefix = serviceName } @@ -122,30 +123,37 @@ func applyDefaultConfig(conf *config.Configuration) { conf.UpdateInterval = "1h" // Update once per hour } + // Connectors if conf.Connectors.GOCDB.Scope == "" { conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future } + // Importers + if conf.Importers.WebAPI.Endpoint == "" { + conf.Importers.WebAPI.Endpoint = "/" + } + + // Exporters + addDefaultConnector := func(enabledList *[]string) { + if len(*enabledList) == 0 { + *enabledList = append(*enabledList, "*") + } + } + if conf.Exporters.WebAPI.Endpoint == "" { conf.Exporters.WebAPI.Endpoint = "/" } - if len(conf.Exporters.WebAPI.EnabledConnectors) == 0 { - conf.Exporters.WebAPI.EnabledConnectors = append(conf.Exporters.WebAPI.EnabledConnectors, "*") - } + addDefaultConnector(&conf.Exporters.WebAPI.EnabledConnectors) if conf.Exporters.CS3API.Endpoint == "" { conf.Exporters.CS3API.Endpoint = "/cs3" } - if len(conf.Exporters.CS3API.EnabledConnectors) == 0 { - conf.Exporters.CS3API.EnabledConnectors = append(conf.Exporters.CS3API.EnabledConnectors, "*") - } + addDefaultConnector(&conf.Exporters.CS3API.EnabledConnectors) if conf.Exporters.SiteLocations.Endpoint == "" { conf.Exporters.SiteLocations.Endpoint = "/loc" } - if len(conf.Exporters.SiteLocations.EnabledConnectors) == 0 { - conf.Exporters.SiteLocations.EnabledConnectors = append(conf.Exporters.SiteLocations.EnabledConnectors, "*") - } + addDefaultConnector(&conf.Exporters.SiteLocations.EnabledConnectors) } // New returns a new Mentix service. diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 3673c09f68..f43c4e6dfd 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -35,6 +35,13 @@ type Configuration struct { UpdateInterval string `mapstructure:"update_interval"` + Importers struct { + WebAPI struct { + Endpoint string `mapstructure:"endpoint"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` + } `mapstructure:"webapi"` + } `mapstructure:"importers"` + Exporters struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f89f411fb0..af655ad984 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,6 +26,9 @@ const ( ) const ( + // ImporterIDWebAPI is the identifier for the WebAPI importer. + ImporterIDWebAPI = "webapi" + // ExporterIDWebAPI is the identifier for the WebAPI exporter. ExporterIDWebAPI = "webapi" // ExporterIDCS3API is the identifier for the CS3API exporter. diff --git a/pkg/mentix/exchange/exporters/reqexporter.go b/pkg/mentix/exchange/exporters/reqexporter.go index 35e7628fce..4e91e39211 100644 --- a/pkg/mentix/exchange/exporters/reqexporter.go +++ b/pkg/mentix/exchange/exporters/reqexporter.go @@ -43,18 +43,13 @@ type BaseRequestExporter struct { } // HandleRequest handles the actual HTTP request. -func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - status, data, err := exporter.handleQuery(req.URL.Query()) - if err == nil { - resp.WriteHeader(status) - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } - } else { - return fmt.Errorf("error while serving API request: %v", err) +func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { + status, respData, err := exporter.handleQuery(req.URL.Query()) + if err != nil { + respData = []byte(err.Error()) } - - return nil + resp.WriteHeader(status) + _, _ = resp.Write(respData) } func (exporter *BaseRequestExporter) handleQuery(params url.Values) (int, []byte, error) { @@ -62,16 +57,16 @@ func (exporter *BaseRequestExporter) handleQuery(params url.Values) (int, []byte exporter.Locker().RLock() defer exporter.Locker().RUnlock() - method := params.Get("action") - switch strings.ToLower(method) { + action := params.Get("action") + switch strings.ToLower(action) { case queryActionDefault: if exporter.defaultActionHandler != nil { return exporter.defaultActionHandler(exporter.MeshData(), params) } default: - return http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", method) + return http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) } - return http.StatusNotImplemented, []byte{}, fmt.Errorf("unhandled query for action '%v'", method) + return http.StatusNotImplemented, []byte{}, fmt.Errorf("unhandled query for action '%v'", action) } diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchange/importers/reqimporter.go index 61a965aa36..5cabe368d3 100644 --- a/pkg/mentix/exchange/importers/reqimporter.go +++ b/pkg/mentix/exchange/importers/reqimporter.go @@ -45,38 +45,33 @@ type BaseRequestImporter struct { } // HandleRequest handles the actual HTTP request. -func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { - meshData, status, data, err := importer.handleQuery(req.URL.Query()) +func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { + meshData, status, respData, err := importer.handleQuery(req.URL.Query()) if err == nil { importer.mergeImportedMeshData(meshData) - - resp.WriteHeader(status) - if _, err := resp.Write(data); err != nil { - return fmt.Errorf("error writing the API request response: %v", err) - } } else { - return fmt.Errorf("error while serving API request: %v", err) + respData = []byte(err.Error()) } - - return nil + resp.WriteHeader(status) + _, _ = resp.Write(respData) } func (importer *BaseRequestImporter) mergeImportedMeshData(meshData *meshdata.MeshData) { - // Data is written, so lock it completely - importer.Locker().Lock() - defer importer.Locker().Unlock() - // Merge the newly imported data with any existing data stored in the importer if meshDataOld := importer.MeshData(); meshDataOld != nil { + // Need to manually lock the data for writing + importer.Locker().Lock() + defer importer.Locker().Unlock() + meshDataOld.Merge(meshData) } else { - importer.SetMeshData(meshData) + importer.SetMeshData(meshData) // SetMeshData will do the locking itself } } func (importer *BaseRequestImporter) handleQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { - method := params.Get("action") - switch strings.ToLower(method) { + action := params.Get("action") + switch strings.ToLower(action) { case queryActionRegisterSite: if importer.registerSiteActionHandler != nil { return importer.registerSiteActionHandler(params) @@ -88,8 +83,8 @@ func (importer *BaseRequestImporter) handleQuery(params url.Values) (*meshdata.M } default: - return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", method) + return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) } - return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", method) + return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", action) } diff --git a/pkg/mentix/exchange/importers/webapi.go b/pkg/mentix/exchange/importers/webapi.go new file mode 100755 index 0000000000..15208a5595 --- /dev/null +++ b/pkg/mentix/exchange/importers/webapi.go @@ -0,0 +1,56 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchange/importers/webapi" +) + +// WebAPIImporter implements the generic Web API importer. +type WebAPIImporter struct { + BaseRequestImporter +} + +// Activate activates the importer. +func (exporter *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseRequestImporter.Activate(conf, log); err != nil { + return err + } + + // Store WebAPI specifics + exporter.SetEndpoint(conf.Importers.WebAPI.Endpoint) + exporter.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) + + exporter.registerSiteActionHandler = webapi.HandleRegisterSiteQuery + exporter.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery + + return nil +} + +// GetName returns the display name of the importer. +func (exporter *WebAPIImporter) GetName() string { + return "WebAPI" +} + +func init() { + registerImporter(config.ImporterIDWebAPI, &WebAPIImporter{}) +} diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go new file mode 100755 index 0000000000..c805b422c0 --- /dev/null +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -0,0 +1,38 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package webapi + +import ( + "net/http" + "net/url" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +// HandleRegisterSiteQuery registers a site. +func HandleRegisterSiteQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { + // TODO: Handlen + Response (ähnlich OCM) + return &meshdata.MeshData{}, http.StatusOK, []byte("registered"), nil +} + +// HandleUnregisterSiteQuery unregisters a site. +func HandleUnregisterSiteQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { + // TODO: Handlen + Response (ähnlich OCM) + return &meshdata.MeshData{}, http.StatusOK, []byte("unregistered"), nil +} diff --git a/pkg/mentix/exchange/reqexchanger.go b/pkg/mentix/exchange/reqexchanger.go index 5322190adf..39446d6d80 100644 --- a/pkg/mentix/exchange/reqexchanger.go +++ b/pkg/mentix/exchange/reqexchanger.go @@ -30,7 +30,7 @@ type RequestExchanger interface { // WantsRequest returns whether the exchanger wants to handle the incoming request. WantsRequest(r *http.Request) bool // HandleRequest handles the actual HTTP request. - HandleRequest(resp http.ResponseWriter, req *http.Request) error + HandleRequest(resp http.ResponseWriter, req *http.Request) } // BaseRequestExchanger implements basic exporter functionality common to all request exporters. diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 23c8a57830..ad282aa92e 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -299,11 +299,9 @@ func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { func (mntx *Mentix) handleRequest(exchangers []exchange.RequestExchanger, w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { // Ask each RequestExchanger if it wants to handle the request - for _, exporter := range exchangers { - if exporter.WantsRequest(r) { - if err := exporter.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling request") - } + for _, exchanger := range exchangers { + if exchanger.WantsRequest(r) { + exchanger.HandleRequest(w, r) } } } From b349a22399d7b013017cb064414d5b50009d6e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 11:35:46 +0100 Subject: [PATCH 18/35] Restructure data handling in exchangers --- pkg/mentix/exchange/exchanger.go | 60 ++++++------------- pkg/mentix/exchange/exporters/exporter.go | 28 +++++++-- pkg/mentix/exchange/exporters/promsd.go | 2 +- pkg/mentix/exchange/importers/importer.go | 44 ++++++++++++-- pkg/mentix/exchange/importers/reqimporter.go | 10 ++-- pkg/mentix/exchange/importers/webapi/query.go | 8 +-- pkg/mentix/mentix.go | 12 ++-- pkg/mentix/meshdata/meshdata.go | 12 ++++ .../meshdata/{meshdataset.go => types.go} | 11 ++-- 9 files changed, 116 insertions(+), 71 deletions(-) rename pkg/mentix/meshdata/{meshdataset.go => types.go} (77%) diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchange/exchanger.go index 0ce0422b35..07eb99ef24 100644 --- a/pkg/mentix/exchange/exchanger.go +++ b/pkg/mentix/exchange/exchanger.go @@ -1,26 +1,20 @@ -/* - * MIT License - * - * Copyright (c) 2020 Daniel Mueller - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. package exchange @@ -32,7 +26,6 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Exchanger is the base interface for importers and exporters. @@ -44,9 +37,6 @@ type Exchanger interface { // Stop stops any running background activities of the exchanger. Stop() - // MeshDataSet returns the mesh data. - MeshData() *meshdata.MeshData - // GetName returns the display name of the exchanger. GetName() string } @@ -60,8 +50,7 @@ type BaseExchanger struct { enabledConnectors []string - meshData *meshdata.MeshData - locker sync.RWMutex + locker sync.RWMutex } // Activate activates the exchanger. @@ -118,19 +107,6 @@ func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) { exchanger.enabledConnectors = connectors } -// MeshDataSet returns the stored mesh data. -func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { - return exchanger.meshData -} - -// SetMeshDataSet sets new mesh data. -func (exchanger *BaseExchanger) SetMeshData(meshData *meshdata.MeshData) { - exchanger.Locker().Lock() - defer exchanger.Locker().Unlock() - - exchanger.meshData = meshData -} - // Locker returns the locking object. func (exchanger *BaseExchanger) Locker() *sync.RWMutex { return &exchanger.locker diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchange/exporters/exporter.go index b8bb44520c..c0de694c30 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchange/exporters/exporter.go @@ -29,17 +29,22 @@ import ( type Exporter interface { exchange.Exchanger + // MeshData returns the mesh data. + MeshData() *meshdata.MeshData + // Update is called whenever the mesh data set has changed to reflect these changes. - Update(meshdata.MeshDataSet) error + Update(meshdata.Map) error } // BaseExporter implements basic exporter functionality common to all exporters. type BaseExporter struct { exchange.BaseExchanger + + meshData *meshdata.MeshData } // Update is called whenever the mesh data set has changed to reflect these changes. -func (exporter *BaseExporter) Update(meshDataSet meshdata.MeshDataSet) error { +func (exporter *BaseExporter) Update(meshDataSet meshdata.Map) error { // Update the stored mesh data set if err := exporter.storeMeshDataSet(meshDataSet); err != nil { return fmt.Errorf("unable to store the mesh data: %v", err) @@ -48,9 +53,9 @@ func (exporter *BaseExporter) Update(meshDataSet meshdata.MeshDataSet) error { return nil } -func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.MeshDataSet) error { +func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.Map) error { // Store the new mesh data set by cloning it and then merging the cloned data into one object - meshDataSetCloned := make(meshdata.MeshDataSet) + meshDataSetCloned := make(meshdata.Map) for connectorID, meshData := range meshDataSet { if !exporter.IsConnectorEnabled(connectorID) { continue @@ -62,7 +67,20 @@ func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.MeshDataSet) } meshDataSetCloned[connectorID] = meshDataCloned } - exporter.SetMeshData(meshdata.MergeMeshDataSet(meshDataSetCloned)) + exporter.SetMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) return nil } + +// MeshData returns the stored mesh data. +func (exporter *BaseExporter) MeshData() *meshdata.MeshData { + return exporter.meshData +} + +// SetMeshData sets new mesh data. +func (exporter *BaseExporter) SetMeshData(meshData *meshdata.MeshData) { + exporter.Locker().Lock() + defer exporter.Locker().Unlock() + + exporter.meshData = meshData +} diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index aa1954a0ce..37c61a4cb0 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -137,7 +137,7 @@ func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log * } // Update is called whenever the mesh data set has changed to reflect these changes. -func (exporter *PrometheusSDExporter) Update(meshDataSet meshdata.MeshDataSet) error { +func (exporter *PrometheusSDExporter) Update(meshDataSet meshdata.Map) error { if err := exporter.BaseExporter.Update(meshDataSet); err != nil { return err } diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go index 21d433ae2a..854467f439 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchange/importers/importer.go @@ -19,23 +19,59 @@ package importers import ( + "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Importer is the interface that all importers must implement. type Importer interface { exchange.Exchanger - // Process is called periodically to perform its tasks. - Process() error + // MeshData returns the vector of imported mesh data. + MeshData() meshdata.Vector + + // Process is called periodically to perform the actual import. + Process([]connectors.Connector) error } // BaseImporter implements basic importer functionality common to all importers. type BaseImporter struct { exchange.BaseExchanger + + meshData meshdata.Vector } -// Process is called periodically to perform its tasks. -func (importer *BaseImporter) Process() error { +// Process is called periodically to perform the actual import. +func (importer *BaseImporter) Process(connectors []connectors.Connector) error { + if meshData := importer.MeshData(); meshData != nil { + // Data is read, so lock it for writing + importer.Locker().RLock() + + for _, connector := range connectors { + if !importer.IsConnectorEnabled(connector.GetID()) { + continue + } + + // TODO: Use Connector to add/remove site/service + } + + importer.Locker().RUnlock() + } + + importer.SetMeshData(nil) return nil } + +// MeshData returns the vector of imported mesh data. +func (importer *BaseImporter) MeshData() meshdata.Vector { + return importer.meshData +} + +// SetMeshData sets the new mesh data vector. +func (importer *BaseImporter) SetMeshData(meshData meshdata.Vector) { + importer.Locker().Lock() + defer importer.Locker().Unlock() + + importer.meshData = meshData +} diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchange/importers/reqimporter.go index 5cabe368d3..77d53cbbe2 100644 --- a/pkg/mentix/exchange/importers/reqimporter.go +++ b/pkg/mentix/exchange/importers/reqimporter.go @@ -33,7 +33,7 @@ const ( queryActionUnregisterSite = "unregister" ) -type queryCallback func(url.Values) (*meshdata.MeshData, int, []byte, error) +type queryCallback func(url.Values) (meshdata.Vector, int, []byte, error) // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { @@ -56,20 +56,20 @@ func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req _, _ = resp.Write(respData) } -func (importer *BaseRequestImporter) mergeImportedMeshData(meshData *meshdata.MeshData) { +func (importer *BaseRequestImporter) mergeImportedMeshData(meshData meshdata.Vector) { // Merge the newly imported data with any existing data stored in the importer - if meshDataOld := importer.MeshData(); meshDataOld != nil { + if importer.meshData != nil { // Need to manually lock the data for writing importer.Locker().Lock() defer importer.Locker().Unlock() - meshDataOld.Merge(meshData) + importer.meshData = append(importer.meshData, meshData...) } else { importer.SetMeshData(meshData) // SetMeshData will do the locking itself } } -func (importer *BaseRequestImporter) handleQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { +func (importer *BaseRequestImporter) handleQuery(params url.Values) (meshdata.Vector, int, []byte, error) { action := params.Get("action") switch strings.ToLower(action) { case queryActionRegisterSite: diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go index c805b422c0..d5d4145494 100755 --- a/pkg/mentix/exchange/importers/webapi/query.go +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -26,13 +26,13 @@ import ( ) // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { +func HandleRegisterSiteQuery(params url.Values) (meshdata.Vector, int, []byte, error) { // TODO: Handlen + Response (ähnlich OCM) - return &meshdata.MeshData{}, http.StatusOK, []byte("registered"), nil + return meshdata.Vector{}, http.StatusOK, []byte("registered"), nil } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(params url.Values) (*meshdata.MeshData, int, []byte, error) { +func HandleUnregisterSiteQuery(params url.Values) (meshdata.Vector, int, []byte, error) { // TODO: Handlen + Response (ähnlich OCM) - return &meshdata.MeshData{}, http.StatusOK, []byte("unregistered"), nil + return meshdata.Vector{}, http.StatusOK, []byte("unregistered"), nil } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index ad282aa92e..9e824c15f8 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -44,7 +44,7 @@ type Mentix struct { importers []importers.Importer exporters []exporters.Exporter - meshDataSet meshdata.MeshDataSet + meshDataSet meshdata.Map updateInterval time.Duration } @@ -83,7 +83,7 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) mntx.updateInterval = duration // Create empty mesh data set - mntx.meshDataSet = make(meshdata.MeshDataSet) + mntx.meshDataSet = make(meshdata.Map) // Log some infos connectorNames := make([]string, len(mntx.connectors)) @@ -223,7 +223,7 @@ loop: func (mntx *Mentix) processImporters() error { for _, importer := range mntx.importers { - if err := importer.Process(); err != nil { + if err := importer.Process(mntx.connectors); err != nil { return fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) } } @@ -231,8 +231,8 @@ func (mntx *Mentix) processImporters() error { return nil } -func (mntx *Mentix) retrieveMeshDataSet() (meshdata.MeshDataSet, error) { - meshDataSet := make(meshdata.MeshDataSet) +func (mntx *Mentix) retrieveMeshDataSet() (meshdata.Map, error) { + meshDataSet := make(meshdata.Map) for _, connector := range mntx.connectors { meshData, err := connector.RetrieveMeshData() @@ -245,7 +245,7 @@ func (mntx *Mentix) retrieveMeshDataSet() (meshdata.MeshDataSet, error) { return meshDataSet, nil } -func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.MeshDataSet) error { +func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error { // Check if mesh data from any connector has changed meshDataChanged := false for connectorID, meshData := range meshDataSet { diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index d2935b1e64..f619bc9c9b 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -23,16 +23,28 @@ import ( "fmt" ) +const ( + // FlagsNone resets all mesh data flags. + FlagsNone = 0 + + // FlagObsolete flags the mesh data for removal. + FlagObsolete = 0x0001 +) + // MeshData holds the entire mesh data managed by Mentix. type MeshData struct { Sites []*Site ServiceTypes []*ServiceType + + Flags int32 `json:"-"` } // Clear removes all saved data, leaving an empty mesh. func (meshData *MeshData) Clear() { meshData.Sites = nil meshData.ServiceTypes = nil + + meshData.Flags = FlagsNone } // Merge merges data from another MeshData instance into this one (w/o checking for duplicates). diff --git a/pkg/mentix/meshdata/meshdataset.go b/pkg/mentix/meshdata/types.go similarity index 77% rename from pkg/mentix/meshdata/meshdataset.go rename to pkg/mentix/meshdata/types.go index 81e16445f4..f7cd17d9bb 100644 --- a/pkg/mentix/meshdata/meshdataset.go +++ b/pkg/mentix/meshdata/types.go @@ -18,11 +18,14 @@ package meshdata -// MeshDataSet represents a set of MeshData objects. -type MeshDataSet = map[string]*MeshData +// Vector represents a vector of MeshData objects. +type Vector = []*MeshData -// MergeMeshDataSet merges all mesh data objects within a set. -func MergeMeshDataSet(meshDataSet MeshDataSet) *MeshData { +// Map represents a map of MeshData objects. +type Map = map[string]*MeshData + +// MergeMeshDataMap merges all mesh data objects within a map. +func MergeMeshDataMap(meshDataSet Map) *MeshData { mergedMeshData := &MeshData{} for _, meshData := range meshDataSet { mergedMeshData.Merge(meshData) From 8d408c6002fc4f1aac51d2a561f4c5805ec53b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 12:11:01 +0100 Subject: [PATCH 19/35] Modify Mentix run-loop to fit importers --- pkg/mentix/exchange/importers/importer.go | 44 +++++++++----- pkg/mentix/exchange/importers/webapi/query.go | 2 +- pkg/mentix/mentix.go | 58 +++++++++++-------- 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go index 854467f439..f31e686b36 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchange/importers/importer.go @@ -19,6 +19,8 @@ package importers import ( + "fmt" + "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchange" "github.com/cs3org/reva/pkg/mentix/meshdata" @@ -31,8 +33,8 @@ type Importer interface { // MeshData returns the vector of imported mesh data. MeshData() meshdata.Vector - // Process is called periodically to perform the actual import. - Process([]connectors.Connector) error + // Process is called periodically to perform the actual import; if data has been imported, true is returned. + Process([]connectors.Connector) (bool, error) } // BaseImporter implements basic importer functionality common to all importers. @@ -42,24 +44,40 @@ type BaseImporter struct { meshData meshdata.Vector } -// Process is called periodically to perform the actual import. -func (importer *BaseImporter) Process(connectors []connectors.Connector) error { - if meshData := importer.MeshData(); meshData != nil { - // Data is read, so lock it for writing - importer.Locker().RLock() +// Process is called periodically to perform the actual import; if data has been imported, true is returned. +func (importer *BaseImporter) Process(connectors []connectors.Connector) (bool, error) { + if importer.meshData == nil { // Nothing to do + return false, nil + } - for _, connector := range connectors { - if !importer.IsConnectorEnabled(connector.GetID()) { - continue - } + // Data is read, so lock it for writing + importer.Locker().RLock() - // TODO: Use Connector to add/remove site/service + for _, connector := range connectors { + if !importer.IsConnectorEnabled(connector.GetID()) { + continue } - importer.Locker().RUnlock() + if err := importer.processMeshData(connector); err != nil { + importer.Locker().RUnlock() + return false, fmt.Errorf("unable to process imported mesh data for connector '%v': %v", connector.GetName(), err) + } } + importer.Locker().RUnlock() + importer.SetMeshData(nil) + return true, nil +} + +func (importer *BaseImporter) processMeshData(connector connectors.Connector) error { + for _, meshData := range importer.meshData { + // TODO: Add/remove stuff + for _, s := range meshData.Sites { + fmt.Println(s.Name) + } + } + return nil } diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go index d5d4145494..12e7a54295 100755 --- a/pkg/mentix/exchange/importers/webapi/query.go +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -28,7 +28,7 @@ import ( // HandleRegisterSiteQuery registers a site. func HandleRegisterSiteQuery(params url.Values) (meshdata.Vector, int, []byte, error) { // TODO: Handlen + Response (ähnlich OCM) - return meshdata.Vector{}, http.StatusOK, []byte("registered"), nil + return meshdata.Vector{&meshdata.MeshData{[]*meshdata.Site{&meshdata.Site{Name: "TEST"}}, []*meshdata.ServiceType{}, 0}}, http.StatusOK, []byte("registered"), nil } // HandleUnregisterSiteQuery unregisters a site. diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 9e824c15f8..a2ef59e7a9 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -194,26 +194,8 @@ loop: } } - // If enough time has passed, process im- and exports of mesh data - if time.Since(updateTimestamp) >= mntx.updateInterval { - // Let all importers do their work first - if err := mntx.processImporters(); err != nil { - mntx.log.Err(err).Msgf("an error occurred while processing the importers: %v", err) - } - - // Retrieve and update the mesh data; if the importers modified any data, these changes will - // be reflected automatically here - meshDataSet, err := mntx.retrieveMeshDataSet() - if err == nil { - if err := mntx.applyMeshDataSet(meshDataSet); err != nil { - mntx.log.Err(err).Msg("failed to apply mesh data") - } - } else { - mntx.log.Err(err).Msg("failed to retrieve mesh data") - } - - updateTimestamp = time.Now() - } + // Perform all regular actions + mntx.tick(&updateTimestamp) time.Sleep(runLoopSleeptime) } @@ -221,14 +203,42 @@ loop: return nil } -func (mntx *Mentix) processImporters() error { +func (mntx *Mentix) tick(updateTimestamp *time.Time) { + // Let all importers do their work first + meshDataUpdated, err := mntx.processImporters() + if err != nil { + mntx.log.Err(err).Msgf("an error occurred while processing the importers: %v", err) + } + + // If mesh data has been imported or enough time has passed, update the stored mesh data and all exporters + if meshDataUpdated || time.Since(*updateTimestamp) >= mntx.updateInterval { + // Retrieve and update the mesh data; if the importers modified any data, these changes will + // be reflected automatically here + meshDataSet, err := mntx.retrieveMeshDataSet() + if err == nil { + if err := mntx.applyMeshDataSet(meshDataSet); err != nil { + mntx.log.Err(err).Msg("failed to apply mesh data") + } + } else { + mntx.log.Err(err).Msg("failed to retrieve mesh data") + } + + *updateTimestamp = time.Now() + } +} + +func (mntx *Mentix) processImporters() (bool, error) { + meshDataUpdated := false + for _, importer := range mntx.importers { - if err := importer.Process(mntx.connectors); err != nil { - return fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) + updated, err := importer.Process(mntx.connectors) + if err != nil { + return false, fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) } + meshDataUpdated = meshDataUpdated || updated } - return nil + return meshDataUpdated, nil } func (mntx *Mentix) retrieveMeshDataSet() (meshdata.Map, error) { From aabc154b989b2d4442dd18e9313c7c969571fd59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 12:29:24 +0100 Subject: [PATCH 20/35] Add support for data updates to connectors --- pkg/mentix/connectors/connector.go | 7 +++++++ pkg/mentix/exchange/importers/importer.go | 22 +++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 716d86d200..bac9abebaf 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -41,6 +41,8 @@ type Connector interface { Activate(conf *config.Configuration, log *zerolog.Logger) error // RetrieveMeshData fetches new mesh data. RetrieveMeshData() (*meshdata.MeshData, error) + // UpdateMeshData updates the provided mesh data on the target side. + UpdateMeshData(data *meshdata.MeshData) error // GetName returns the display name of the connector. GetName() string @@ -82,6 +84,11 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo return nil } +// UpdateMeshData updates the provided mesh data on the target side. +func (connector *BaseConnector) UpdateMeshData(data *meshdata.MeshData) error { + return fmt.Errorf("the connector doesn't support updating of mesh data") +} + // AvailableConnectors returns a list of all connectors that are enabled in the configuration. func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { entries, err := registeredConnectors.EntriesByID(conf.EnabledConnectors) diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchange/importers/importer.go index f31e686b36..7043d15f74 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchange/importers/importer.go @@ -20,6 +20,7 @@ package importers import ( "fmt" + "strings" "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchange" @@ -50,31 +51,34 @@ func (importer *BaseImporter) Process(connectors []connectors.Connector) (bool, return false, nil } - // Data is read, so lock it for writing - importer.Locker().RLock() + var processErrs []string + // Data is read, so lock it for writing during the loop + importer.Locker().RLock() for _, connector := range connectors { if !importer.IsConnectorEnabled(connector.GetID()) { continue } if err := importer.processMeshData(connector); err != nil { - importer.Locker().RUnlock() - return false, fmt.Errorf("unable to process imported mesh data for connector '%v': %v", connector.GetName(), err) + processErrs = append(processErrs, fmt.Sprintf("unable to process imported mesh data for connector '%v': %v", connector.GetName(), err)) } } - importer.Locker().RUnlock() importer.SetMeshData(nil) - return true, nil + + var err error + if len(processErrs) != 0 { + err = fmt.Errorf(strings.Join(processErrs, "; ")) + } + return true, err } func (importer *BaseImporter) processMeshData(connector connectors.Connector) error { for _, meshData := range importer.meshData { - // TODO: Add/remove stuff - for _, s := range meshData.Sites { - fmt.Println(s.Name) + if err := connector.UpdateMeshData(meshData); err != nil { + return fmt.Errorf("error while updating mesh data: %v", err) } } From 9920e058b5c272080d5fef475254742a587bf5ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 14:48:46 +0100 Subject: [PATCH 21/35] Finish WebAPI importer --- pkg/mentix/exchange/importers/reqimporter.go | 12 +++-- pkg/mentix/exchange/importers/webapi/query.go | 48 ++++++++++++++++--- pkg/mentix/mentix.go | 2 + pkg/mentix/meshdata/site.go | 10 ++++ pkg/mentix/network/utils.go | 33 +++++++++++++ 5 files changed, 94 insertions(+), 11 deletions(-) diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchange/importers/reqimporter.go index 77d53cbbe2..3a65327b02 100644 --- a/pkg/mentix/exchange/importers/reqimporter.go +++ b/pkg/mentix/exchange/importers/reqimporter.go @@ -20,6 +20,7 @@ package importers import ( "fmt" + "io/ioutil" "net/http" "net/url" "strings" @@ -33,7 +34,7 @@ const ( queryActionUnregisterSite = "unregister" ) -type queryCallback func(url.Values) (meshdata.Vector, int, []byte, error) +type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { @@ -46,7 +47,8 @@ type BaseRequestImporter struct { // HandleRequest handles the actual HTTP request. func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { - meshData, status, respData, err := importer.handleQuery(req.URL.Query()) + body, _ := ioutil.ReadAll(req.Body) + meshData, status, respData, err := importer.handleQuery(body, req.URL.Query()) if err == nil { importer.mergeImportedMeshData(meshData) } else { @@ -69,17 +71,17 @@ func (importer *BaseRequestImporter) mergeImportedMeshData(meshData meshdata.Vec } } -func (importer *BaseRequestImporter) handleQuery(params url.Values) (meshdata.Vector, int, []byte, error) { +func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { action := params.Get("action") switch strings.ToLower(action) { case queryActionRegisterSite: if importer.registerSiteActionHandler != nil { - return importer.registerSiteActionHandler(params) + return importer.registerSiteActionHandler(data, params) } case queryActionUnregisterSite: if importer.unregisterSiteActionHandler != nil { - return importer.unregisterSiteActionHandler(params) + return importer.unregisterSiteActionHandler(data, params) } default: diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go index 12e7a54295..15afcb72a4 100755 --- a/pkg/mentix/exchange/importers/webapi/query.go +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -19,20 +19,56 @@ package webapi import ( + "encoding/json" + "fmt" "net/http" "net/url" "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/network" ) +func decodeQueryData(data []byte) (*meshdata.MeshData, error) { + site := &meshdata.Site{} + if err := json.Unmarshal(data, site); err != nil { + return nil, err + } + + // Verify that the absolute minimum of information is provided + if site.Name == "" { + return nil, fmt.Errorf("site name missing") + } + if site.Domain == "" && site.Homepage == "" { + return nil, fmt.Errorf("site URL missing") + } + + // Infer missing data + if site.Homepage == "" { + site.Homepage = fmt.Sprintf("http://www.%v", site.Domain) + } else if site.Domain == "" { + if URL, err := url.Parse(site.Homepage); err == nil { + site.Domain = network.ExtractDomainFromURL(URL) + } + } + + return &meshdata.MeshData{Sites: []*meshdata.Site{site}}, nil +} + +func handleQuery(data []byte, params url.Values, flags int32, msg string) (meshdata.Vector, int, []byte, error) { + meshData, err := decodeQueryData(data) + if err != nil { + return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil + } + meshData.Flags = flags + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].GetID()}), nil +} + // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(params url.Values) (meshdata.Vector, int, []byte, error) { - // TODO: Handlen + Response (ähnlich OCM) - return meshdata.Vector{&meshdata.MeshData{[]*meshdata.Site{&meshdata.Site{Name: "TEST"}}, []*meshdata.ServiceType{}, 0}}, http.StatusOK, []byte("registered"), nil +func HandleRegisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + return handleQuery(data, params, meshdata.FlagsNone, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(params url.Values) (meshdata.Vector, int, []byte, error) { - // TODO: Handlen + Response (ähnlich OCM) - return meshdata.Vector{}, http.StatusOK, []byte("unregistered"), nil +func HandleUnregisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + return handleQuery(data, params, meshdata.FlagObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index a2ef59e7a9..b64ab97002 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -293,6 +293,8 @@ func (mntx *Mentix) GetRequestExporters() []exchange.RequestExchanger { // RequestHandler handles any incoming HTTP requests by asking each RequestExchanger whether it wants to // handle the request (usually based on the relative URL path). func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + log := appctx.GetLogger(r.Context()) switch r.Method { diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index f19fd929b3..b2351bab4c 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -18,6 +18,10 @@ package meshdata +import ( + "fmt" +) + const ( // SiteTypeScienceMesh flags a site as being part of the mesh. SiteTypeScienceMesh = iota @@ -47,3 +51,9 @@ type Site struct { Services []*Service Properties map[string]string } + +// GetID generates a unique ID for the site; the following fields are used for this: +// Name, Domain +func (site *Site) GetID() string { + return fmt.Sprintf("%s::[%s]", site.Domain, site.Name) +} diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index a9e14d06ef..fcd23163ab 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -19,16 +19,22 @@ package network import ( + "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" p "path" + "strings" ) // URLParams holds Key-Value URL parameters; it is a simpler form of url.Values. type URLParams map[string]string +// ResponseParams holds parameters of an HTTP response. +type ResponseParams map[string]interface{} + // GenerateURL creates a URL object from a host, path and optional parameters. func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { fullURL, err := url.Parse(host) @@ -67,3 +73,30 @@ func ReadEndpoint(host string, path string, params URLParams) ([]byte, error) { body, _ := ioutil.ReadAll(resp.Body) return body, nil } + +// CreateResponse creates a generic HTTP response in JSON format. +func CreateResponse(msg string, params ResponseParams) []byte { + if params == nil { + params = make(map[string]interface{}) + } + params["message"] = msg + + jsonData, _ := json.MarshalIndent(params, "", "\t") + return jsonData +} + +// ExtractDomainFromURL extracts the domain name (domain.tld) from a URL. +func ExtractDomainFromURL(hostURL *url.URL) string { + // Remove host port if present + host, _, err := net.SplitHostPort(hostURL.Host) + if err != nil { + host = hostURL.Host + } + + // Remove subdomain + if idx := strings.Index(host, "."); idx != -1 { + host = host[idx+1:] + } + + return host +} From 33078d613fd6db44e82d4f97df1100a19ef24c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 15:47:56 +0100 Subject: [PATCH 22/35] Add mesh data updates support to the localfile connector --- pkg/mentix/connectors/localfile.go | 44 +++++++-- pkg/mentix/exchange/importers/webapi/query.go | 2 +- pkg/mentix/mentix.go | 4 + pkg/mentix/meshdata/meshdata.go | 91 ++++++++++++++++++- pkg/mentix/meshdata/site.go | 47 +++++++++- pkg/mentix/network/utils.go | 12 ++- 6 files changed, 180 insertions(+), 20 deletions(-) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index bb5cb916e2..f82827eab0 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -49,26 +49,25 @@ func (connector *LocalFileConnector) Activate(conf *config.Configuration, log *z return fmt.Errorf("no file configured") } + // Create an empty file if it doesn't exist + if _, err := os.Stat(connector.filePath); os.IsNotExist(err) { + ioutil.WriteFile(connector.filePath, []byte("[]"), os.ModePerm) + } + return nil } // RetrieveMeshData fetches new mesh data. func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { - meshData := new(meshdata.MeshData) - - jsonFile, err := os.Open(connector.filePath) - if err != nil { - return nil, fmt.Errorf("unable to open input file '%v': %v", connector.filePath, err) - } - defer jsonFile.Close() + meshData := &meshdata.MeshData{} - jsonData, err := ioutil.ReadAll(jsonFile) + jsonData, err := ioutil.ReadFile(connector.filePath) if err != nil { - return nil, fmt.Errorf("unable to read input file '%v': %v", connector.filePath, err) + return nil, fmt.Errorf("unable to read file '%v': %v", connector.filePath, err) } if err := json.Unmarshal(jsonData, &meshData.Sites); err != nil { - return nil, fmt.Errorf("invalid input file '%v': %v", connector.filePath, err) + return nil, fmt.Errorf("invalid file '%v': %v", connector.filePath, err) } // Update the site types, as these are not part of the JSON data @@ -77,6 +76,31 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err return meshData, nil } +// UpdateMeshData updates the provided mesh data on the target side. +func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshData) error { + meshData, err := connector.RetrieveMeshData() + if err != nil { + // Ignore errors and start with an empty data set + meshData = &meshdata.MeshData{} + } + + if (updatedData.Flags & meshdata.FlagObsolete) == meshdata.FlagObsolete { + // Remove data by unmerging + meshData.Unmerge(updatedData) + } else { + // Add/update data by merging + meshData.Merge(updatedData) + } + + // Write the updated sites back to the file + jsonData, _ := json.MarshalIndent(meshData.Sites, "", "\t") + if err := ioutil.WriteFile(connector.filePath, jsonData, os.ModePerm); err != nil { + return fmt.Errorf("unable to write file '%v': %v", connector.filePath, err) + } + + return nil +} + func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { for _, site := range meshData.Sites { site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go index 15afcb72a4..4657ccdbb0 100755 --- a/pkg/mentix/exchange/importers/webapi/query.go +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -47,7 +47,7 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { site.Homepage = fmt.Sprintf("http://www.%v", site.Domain) } else if site.Domain == "" { if URL, err := url.Parse(site.Homepage); err == nil { - site.Domain = network.ExtractDomainFromURL(URL) + site.Domain = network.ExtractDomainFromURL(URL, false) } } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index b64ab97002..981100189e 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -236,6 +236,10 @@ func (mntx *Mentix) processImporters() (bool, error) { return false, fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) } meshDataUpdated = meshDataUpdated || updated + + if updated { + mntx.log.Debug().Msgf("mesh data imported from '%v'", importer.GetName()) + } } return meshDataUpdated, nil diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index f619bc9c9b..5418783986 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -21,6 +21,7 @@ package meshdata import ( "encoding/json" "fmt" + "strings" ) const ( @@ -47,10 +48,94 @@ func (meshData *MeshData) Clear() { meshData.Flags = FlagsNone } -// Merge merges data from another MeshData instance into this one (w/o checking for duplicates). +// AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. +func (meshData *MeshData) AddSite(site *Site) { + if siteExisting := meshData.FindSite(site.GetID()); siteExisting != nil { + *siteExisting = *site + } else { + meshData.Sites = append(meshData.Sites, site) + } +} + +// RemoveSite removes the provided site. +func (meshData *MeshData) RemoveSite(id string) { + if site := meshData.FindSite(id); site != nil { + for idx, siteExisting := range meshData.Sites { + if siteExisting == site { + lastIdx := len(meshData.Sites) - 1 + meshData.Sites[idx] = meshData.Sites[lastIdx] + meshData.Sites[lastIdx] = nil + meshData.Sites = meshData.Sites[:lastIdx] + break + } + } + } +} + +// FindSite searches for a site with the given ID. +func (meshData *MeshData) FindSite(id string) *Site { + for _, site := range meshData.Sites { + if strings.EqualFold(site.GetID(), id) { + return site + } + } + return nil +} + +// AddServiceType adds a new service type; if a type with the same name already exists, the existing one is overwritten. +func (meshData *MeshData) AddServiceType(serviceType *ServiceType) { + if svcTypeExisting := meshData.FindServiceType(serviceType.Name); svcTypeExisting != nil { + *svcTypeExisting = *serviceType + } else { + meshData.ServiceTypes = append(meshData.ServiceTypes, serviceType) + } +} + +// RemoveServiceType removes the provided service type. +func (meshData *MeshData) RemoveServiceType(name string) { + if serviceType := meshData.FindServiceType(name); serviceType != nil { + for idx, svcTypeExisting := range meshData.ServiceTypes { + if svcTypeExisting == serviceType { + lastIdx := len(meshData.ServiceTypes) - 1 + meshData.ServiceTypes[idx] = meshData.ServiceTypes[lastIdx] + meshData.ServiceTypes[lastIdx] = nil + meshData.ServiceTypes = meshData.ServiceTypes[:lastIdx] + break + } + } + } +} + +// FindServiceType searches for a service type with the given name. +func (meshData *MeshData) FindServiceType(name string) *ServiceType { + for _, serviceType := range meshData.ServiceTypes { + if strings.EqualFold(serviceType.Name, name) { + return serviceType + } + } + return nil +} + +// Merge merges data from another MeshData instance into this one. func (meshData *MeshData) Merge(inData *MeshData) { - meshData.Sites = append(meshData.Sites, inData.Sites...) - meshData.ServiceTypes = append(meshData.ServiceTypes, inData.ServiceTypes...) + for _, site := range inData.Sites { + meshData.AddSite(site) + } + + for _, serviceType := range inData.ServiceTypes { + meshData.AddServiceType(serviceType) + } +} + +// Merge removes data from another MeshData instance from this one. +func (meshData *MeshData) Unmerge(inData *MeshData) { + for _, site := range inData.Sites { + meshData.RemoveSite(site.GetID()) + } + + for _, serviceType := range inData.ServiceTypes { + meshData.RemoveServiceType(serviceType.Name) + } } // ToJSON converts the data to JSON. diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index b2351bab4c..02e7d91d0f 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -20,6 +20,10 @@ package meshdata import ( "fmt" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/network" ) const ( @@ -52,8 +56,49 @@ type Site struct { Properties map[string]string } +// AddService adds a new service; if a service with the same name already exists, the existing one is overwritten. +func (site *Site) AddService(service *Service) { + if serviceExisting := site.FindService(service.Name); serviceExisting != nil { + *service = *serviceExisting + } else { + site.Services = append(site.Services, service) + } +} + +// RemoveService removes the provided service. +func (site *Site) RemoveService(name string) { + if service := site.FindService(name); service != nil { + for idx, serviceExisting := range site.Services { + if serviceExisting == service { + lastIdx := len(site.Services) - 1 + site.Services[idx] = site.Services[lastIdx] + site.Services[lastIdx] = nil + site.Services = site.Services[:lastIdx] + break + } + } + } +} + +// FindService searches for a service with the given name. +func (site *Site) FindService(name string) *Service { + for _, service := range site.Services { + if strings.EqualFold(service.Name, name) { + return service + } + } + return nil +} + // GetID generates a unique ID for the site; the following fields are used for this: // Name, Domain func (site *Site) GetID() string { - return fmt.Sprintf("%s::[%s]", site.Domain, site.Name) + host := site.Domain + if site.Homepage != "" { + if hostURL, err := url.Parse(site.Homepage); err == nil { + host = network.ExtractDomainFromURL(hostURL, true) + } + } + + return fmt.Sprintf("%s::[%s]", host, site.Name) } diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index fcd23163ab..52be0075cf 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -85,17 +85,19 @@ func CreateResponse(msg string, params ResponseParams) []byte { return jsonData } -// ExtractDomainFromURL extracts the domain name (domain.tld) from a URL. -func ExtractDomainFromURL(hostURL *url.URL) string { +// ExtractDomainFromURL extracts the domain name (domain.tld or subdomain.domain.tld) from a URL. +func ExtractDomainFromURL(hostURL *url.URL, keepSubdomain bool) string { // Remove host port if present host, _, err := net.SplitHostPort(hostURL.Host) if err != nil { host = hostURL.Host } - // Remove subdomain - if idx := strings.Index(host, "."); idx != -1 { - host = host[idx+1:] + if !keepSubdomain { + // Remove subdomain + if idx := strings.Index(host, "."); idx != -1 { + host = host[idx+1:] + } } return host From 3c051b7664cbe35dee5c303387136a51b6f217eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 3 Nov 2020 15:58:22 +0100 Subject: [PATCH 23/35] Small improvements --- pkg/mentix/connectors/localfile.go | 9 +++++++-- pkg/mentix/exchange/exporters/promsd.go | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index f82827eab0..1da9f164bc 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -23,6 +23,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "github.com/rs/zerolog" @@ -49,9 +50,13 @@ func (connector *LocalFileConnector) Activate(conf *config.Configuration, log *z return fmt.Errorf("no file configured") } + // Create the file directory if necessary + dir := filepath.Dir(connector.filePath) + _ = os.MkdirAll(dir, 0755) + // Create an empty file if it doesn't exist if _, err := os.Stat(connector.filePath); os.IsNotExist(err) { - ioutil.WriteFile(connector.filePath, []byte("[]"), os.ModePerm) + _ = ioutil.WriteFile(connector.filePath, []byte("[]"), 0755) } return nil @@ -94,7 +99,7 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa // Write the updated sites back to the file jsonData, _ := json.MarshalIndent(meshData.Sites, "", "\t") - if err := ioutil.WriteFile(connector.filePath, jsonData, os.ModePerm); err != nil { + if err := ioutil.WriteFile(connector.filePath, jsonData, 0755); err != nil { return fmt.Errorf("unable to write file '%v': %v", connector.filePath, err) } diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index 37c61a4cb0..4179746fc5 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -93,7 +93,7 @@ func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Config } // Create the output directory for the target file so it exists when exporting - if err := os.MkdirAll(filepath.Dir(outputFilename), os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Dir(outputFilename), 0755); err != nil { return fmt.Errorf("unable to create output directory tree: %v", err) } } @@ -125,7 +125,7 @@ func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log * // Create all output directories for _, creator := range exporter.scrapeCreators { - if err := os.MkdirAll(filepath.Dir(creator.outputFilename), os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Dir(creator.outputFilename), 0755); err != nil { return fmt.Errorf("unable to create directory tree: %v", err) } } @@ -200,7 +200,7 @@ func (exporter *PrometheusSDExporter) exportScrapeConfig(outputFilename string, } // Write the data to disk - if err := ioutil.WriteFile(outputFilename, data, os.ModePerm); err != nil { + if err := ioutil.WriteFile(outputFilename, data, 0755); err != nil { return fmt.Errorf("unable to write scrape config '%v': %v", outputFilename, err) } From da6848e0e4d51c7c15e0ff16818e7698d20ff23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 4 Nov 2020 12:27:13 +0100 Subject: [PATCH 24/35] Acceptance tests of imported mesh data --- pkg/mentix/exchange/importers/webapi/query.go | 22 ++----- pkg/mentix/meshdata/meshdata.go | 32 +++++++++ pkg/mentix/meshdata/service.go | 65 +++++++++++++++++++ pkg/mentix/meshdata/site.go | 37 +++++++++++ 4 files changed, 139 insertions(+), 17 deletions(-) diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchange/importers/webapi/query.go index 4657ccdbb0..bc99011a80 100755 --- a/pkg/mentix/exchange/importers/webapi/query.go +++ b/pkg/mentix/exchange/importers/webapi/query.go @@ -34,24 +34,12 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { return nil, err } - // Verify that the absolute minimum of information is provided - if site.Name == "" { - return nil, fmt.Errorf("site name missing") + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + if err := meshData.Verify(); err != nil { + return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) } - if site.Domain == "" && site.Homepage == "" { - return nil, fmt.Errorf("site URL missing") - } - - // Infer missing data - if site.Homepage == "" { - site.Homepage = fmt.Sprintf("http://www.%v", site.Domain) - } else if site.Domain == "" { - if URL, err := url.Parse(site.Homepage); err == nil { - site.Domain = network.ExtractDomainFromURL(URL, false) - } - } - - return &meshdata.MeshData{Sites: []*meshdata.Site{site}}, nil + meshData.InferMissingData() + return meshData, nil } func handleQuery(data []byte, params url.Values, flags int32, msg string) (meshdata.Vector, int, []byte, error) { diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 5418783986..db00a4191d 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -138,6 +138,38 @@ func (meshData *MeshData) Unmerge(inData *MeshData) { } } +// Verify checks if the mesh data is valid. +func (meshData *MeshData) Verify() error { + // Verify all sites + for _, site := range meshData.Sites { + if err := site.Verify(); err != nil { + return err + } + } + + // Verify all service types + for _, serviceType := range meshData.ServiceTypes { + if err := serviceType.Verify(); err != nil { + return err + } + } + + return nil +} + +// InferMissingData infers missing data from other data where possible. +func (meshData *MeshData) InferMissingData() { + // Infer missing site data + for _, site := range meshData.Sites { + site.InferMissingData() + } + + // Infer missing service type data + for _, serviceType := range meshData.ServiceTypes { + serviceType.InferMissingData() + } +} + // ToJSON converts the data to JSON. func (meshData *MeshData) ToJSON() (string, error) { data, err := json.MarshalIndent(meshData, "", "\t") diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index a032c426bb..a77d84d5bf 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,6 +18,13 @@ package meshdata +import ( + "fmt" + "net/url" + + "github.com/cs3org/reva/pkg/mentix/network" +) + // Service represents a service managed by Mentix. type Service struct { *ServiceEndpoint @@ -26,12 +33,51 @@ type Service struct { AdditionalEndpoints []*ServiceEndpoint } +// InferMissingData infers missing data from other data where possible. +func (service *Service) InferMissingData() { + service.ServiceEndpoint.InferMissingData() + + // Infer missing data + if service.Host == "" { + if serviceURL, err := url.Parse(service.URL); err == nil { + service.Host = network.ExtractDomainFromURL(serviceURL, true) + } + } +} + +// Verify checks if the service data is valid. +func (service *Service) Verify() error { + if err := service.ServiceEndpoint.Verify(); err != nil { + return err + } + + return nil +} + // ServiceType represents a service type managed by Mentix. type ServiceType struct { Name string Description string } +// InferMissingData infers missing data from other data where possible. +func (serviceType *ServiceType) InferMissingData() { + // Infer missing data + if serviceType.Description == "" { + serviceType.Description = serviceType.Name + } +} + +// Verify checks if the service type data is valid. +func (serviceType *ServiceType) Verify() error { + // Verify data + if serviceType.Name == "" { + return fmt.Errorf("service type name missing") + } + + return nil +} + // ServiceEndpoint represents a service endpoint managed by Mentix. type ServiceEndpoint struct { Type *ServiceType @@ -40,3 +86,22 @@ type ServiceEndpoint struct { IsMonitored bool Properties map[string]string } + +// InferMissingData infers missing data from other data where possible. +func (serviceEndpoint *ServiceEndpoint) InferMissingData() { +} + +// Verify checks if the service endpoint data is valid. +func (serviceEndpoint *ServiceEndpoint) Verify() error { + if serviceEndpoint.Type == nil { + return fmt.Errorf("service endpoint type missing") + } + if serviceEndpoint.Name == "" { + return fmt.Errorf("service endpoint name missing") + } + if serviceEndpoint.URL == "" { + return fmt.Errorf("service endpoint URL missing") + } + + return nil +} diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 02e7d91d0f..92ae753db9 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -90,6 +90,43 @@ func (site *Site) FindService(name string) *Service { return nil } +// Verify checks if the site data is valid. +func (site *Site) Verify() error { + // Verify data + if site.Name == "" { + return fmt.Errorf("site name missing") + } + if site.Domain == "" && site.Homepage == "" { + return fmt.Errorf("site URL missing") + } + + // Verify services + for _, service := range site.Services { + if err := service.Verify(); err != nil { + return err + } + } + + return nil +} + +// InferMissingData infers missing data from other data where possible. +func (site *Site) InferMissingData() { + // Infer missing data + if site.Homepage == "" { + site.Homepage = fmt.Sprintf("http://www.%v", site.Domain) + } else if site.Domain == "" { + if URL, err := url.Parse(site.Homepage); err == nil { + site.Domain = network.ExtractDomainFromURL(URL, false) + } + } + + // Infer missing for services + for _, service := range site.Services { + service.InferMissingData() + } +} + // GetID generates a unique ID for the site; the following fields are used for this: // Name, Domain func (site *Site) GetID() string { From 8f6283adb8829beb5db9c5ea49ab2e7067628170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 4 Nov 2020 12:46:09 +0100 Subject: [PATCH 25/35] Update documentation --- .../config/http/services/mentix/_index.md | 16 ++++++--- .../http/services/mentix/webapi/_index.md | 36 ++++++++++++++++++- examples/mentix/mentix.toml | 11 ++++-- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index de944fad3c..0f696886a2 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -7,7 +7,7 @@ description: > --- {{% pageinfo %}} -Mentix (_**Me**sh E**nti**ty E**x**porter_) is a service to read mesh topology data from one or more sources (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. +Mentix (_**Me**sh E**nti**ty E**x**changer_) is a service to read and write mesh topology data to and from one or more sources (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. {{% /pageinfo %}} {{% dir name="prefix" type="string" default="mentix" %}} @@ -26,7 +26,7 @@ update_interval = "15m" {{< /highlight >}} {{% /dir %}} -#### Connectors +## Connectors Mentix is decoupled from the actual sources of the mesh data by using so-called _connectors_. A connector is used to gather the data from a certain source, which are then converted into Mentix' own internal format. _Supported connectors:_ @@ -35,9 +35,17 @@ _Supported connectors:_ The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)). - **localfile** -The [localfile](localfile) connector reads sites from a local JSON file. The file must contain an array of sites adhering to the `meshdata.Site` structure. +The [localfile](localfile) connector reads sites from a local JSON file. The file must contain an array of sites adhering to the `meshdata.Site` structure. + +## Importers +Mentix can import mesh data from various sources and write it to one or more targets through the corresponding connectors. -#### Exporters +__Supported importers:__ + +- **webapi** +Mentix can import mesh data via an HTTP endpoint using the `webapi` importer. Data can be sent to the configured relative endpoint (see [here](webapi)). + +## Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. __Supported exporters:__ diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index af61e94673..9cf4eaea6c 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -7,9 +7,43 @@ description: > --- {{% pageinfo %}} -The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. +The WebAPI of Mentix supports both importing and exporting of mesh data via an HTTP endpoint. Both the im- and exporter are configured separately. {{% /pageinfo %}} +## Importer + +The WebAPI importer receives a single _plain_ Mentix site through an HTTP `POST` request; service types are currently not supported. + +The importer supports two actions that must be passed in the URL: +``` +https://sciencemesh.example.com/mentix/webapi/?action= +``` +Currently, the following actions are supported: +- `register`: Registers a new site +- `unregister`: Unregisters an existing site + +For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. + +{{% dir name="endpoint" type="string" default="/" %}} +The endpoint where the mesh data can be sent to. +{{< highlight toml >}} +[http.services.mentix.importers.webapi] +endpoint = "/data" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="" %}} +A list of all enabled connectors for the importer. Must always be provided. +{{< highlight toml >}} +[http.services.mentix.importers.webapi] +enabled_connectors = ["localfile"] +{{< /highlight >}} +{{% /dir %}} + +## Exporter + +The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. + {{% dir name="endpoint" type="string" default="/" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index dfd5359fb2..e82c6f8ee1 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -25,9 +25,14 @@ endpoint = "/" # If this setting is omitted, all connectors will be used as data sources enabled_connectors = ["gocdb"] +# Enable the WebAPI importer +[http.services.mentix.importers.webapi] +# For importers, this is obligatory; the connectors will be used as the target for data updates +enabled_connectors = ["localfile"] + # Configure the Prometheus Service Discovery: -# [http.services.mentix.exporters.promsd] +[http.services.mentix.exporters.promsd] # The following files must be made available to Prometheus. # They can then be used as the file_sd source of a job. -# metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json" -# blackbox_output_file = "/usr/share/prom/sciencemesh_blackbox.json" +metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json" +blackbox_output_file = "/usr/share/prom/sciencemesh_blackbox.json" From d40f264c3f26744a82b61a7cb332a535cabc1b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 5 Nov 2020 11:47:23 +0100 Subject: [PATCH 26/35] Add Entity type to streamline code --- pkg/mentix/connectors/connector.go | 51 +---------------- .../registry.go => connectors/connectors.go} | 44 +++++++------- pkg/mentix/connectors/gocdb.go | 9 ++- pkg/mentix/connectors/localfile.go | 9 ++- pkg/mentix/entity/entity.go | 28 +++++++++ pkg/mentix/entity/registry.go | 57 +++++++++++++++++++ pkg/mentix/exchange/exchanger.go | 6 +- pkg/mentix/exchange/exporters/cs3api.go | 7 ++- pkg/mentix/exchange/exporters/exporters.go | 17 ++---- pkg/mentix/exchange/exporters/promsd.go | 7 ++- .../exchange/exporters/sitelocations.go | 7 ++- pkg/mentix/exchange/exporters/webapi.go | 7 ++- pkg/mentix/exchange/importers/importers.go | 14 ++--- pkg/mentix/exchange/importers/webapi.go | 7 ++- 14 files changed, 164 insertions(+), 106 deletions(-) rename pkg/mentix/{util/registry/registry.go => connectors/connectors.go} (51%) create mode 100644 pkg/mentix/entity/entity.go create mode 100644 pkg/mentix/entity/registry.go diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index bac9abebaf..3a5181a049 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -24,18 +24,13 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/util/registry" -) - -var ( - registeredConnectors = registry.NewRegistry() ) // Connector is the interface that all connectors must implement. type Connector interface { - // GetID returns the ID of the connector. - GetID() string + entity.Entity // Activate activates a connector. Activate(conf *config.Configuration, log *zerolog.Logger) error @@ -43,32 +38,14 @@ type Connector interface { RetrieveMeshData() (*meshdata.MeshData, error) // UpdateMeshData updates the provided mesh data on the target side. UpdateMeshData(data *meshdata.MeshData) error - - // GetName returns the display name of the connector. - GetName() string } // BaseConnector implements basic connector functionality common to all connectors. type BaseConnector struct { - id string - conf *config.Configuration log *zerolog.Logger } -// GetID returns the ID of the connector. -func (connector *BaseConnector) GetID() string { - return connector.id -} - -// SetID sets the ID of the connector. -func (connector *BaseConnector) SetID(id string) { - // The ID can only be set once - if connector.id == "" { - connector.id = id - } -} - // Activate activates the connector. func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { @@ -88,27 +65,3 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo func (connector *BaseConnector) UpdateMeshData(data *meshdata.MeshData) error { return fmt.Errorf("the connector doesn't support updating of mesh data") } - -// AvailableConnectors returns a list of all connectors that are enabled in the configuration. -func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { - entries, err := registeredConnectors.EntriesByID(conf.EnabledConnectors) - if err != nil { - return nil, err - } - - connectors := make([]Connector, 0, len(entries)) - for _, entry := range entries { - connectors = append(connectors, entry.(Connector)) - } - - // At least one connector must be configured - if len(connectors) == 0 { - return nil, fmt.Errorf("no connectors available") - } - - return connectors, nil -} - -func registerConnector(connector Connector) { - registeredConnectors.Register(connector.GetID(), connector) -} diff --git a/pkg/mentix/util/registry/registry.go b/pkg/mentix/connectors/connectors.go similarity index 51% rename from pkg/mentix/util/registry/registry.go rename to pkg/mentix/connectors/connectors.go index 9f362cb6ca..3ba5e5b159 100644 --- a/pkg/mentix/util/registry/registry.go +++ b/pkg/mentix/connectors/connectors.go @@ -16,36 +16,32 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package registry +package connectors -import "fmt" +import ( + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" +) -// Registry represents a simple id->entry map. -type Registry struct { - Entries map[string]interface{} -} +var ( + registeredConnectors = entity.NewRegistry() +) -// Register registers a new entry. -func (r *Registry) Register(id string, entry interface{}) { - r.Entries[id] = entry -} +// AvailableConnectors returns a list of all connectors that are enabled in the configuration. +func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { + entities, err := registeredConnectors.FindEntities(conf.EnabledConnectors, true, true) + if err != nil { + return nil, err + } -// EntriesByID returns all entries matching the provided IDs. If an entry with a certain ID doesn't exist, an error is returned. -func (r *Registry) EntriesByID(ids []string) ([]interface{}, error) { - var entries []interface{} - for _, id := range ids { - if entry, ok := r.Entries[id]; ok { - entries = append(entries, entry) - } else { - return nil, fmt.Errorf("no entry with ID '%v' registered", id) - } + connectors := make([]Connector, 0, len(entities)) + for _, entry := range entities { + connectors = append(connectors, entry.(Connector)) } - return entries, nil + return connectors, nil } -func NewRegistry() *Registry { - return &Registry{ - Entries: make(map[string]interface{}), - } +func registerConnector(connector Connector) { + registeredConnectors.Register(connector) } diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 2663d1933d..9ac1a4399c 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -258,13 +258,16 @@ func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint return svcURL, nil } +// GetID returns the ID of the connector. +func (connector *GOCDBConnector) GetID() string { + return config.ConnectorIDGOCDB +} + // GetName returns the display name of the connector. func (connector *GOCDBConnector) GetName() string { return "GOCDB" } func init() { - connector := &GOCDBConnector{} - connector.SetID(config.ConnectorIDGOCDB) - registerConnector(connector) + registerConnector(&GOCDBConnector{}) } diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 1da9f164bc..15918375e3 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -112,13 +112,16 @@ func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { } } +// GetID returns the ID of the connector. +func (connector *LocalFileConnector) GetID() string { + return config.ConnectorIDLocalFile +} + // GetName returns the display name of the connector. func (connector *LocalFileConnector) GetName() string { return "Local file" } func init() { - connector := &LocalFileConnector{} - connector.SetID(config.ConnectorIDLocalFile) - registerConnector(connector) + registerConnector(&LocalFileConnector{}) } diff --git a/pkg/mentix/entity/entity.go b/pkg/mentix/entity/entity.go new file mode 100644 index 0000000000..269dccc643 --- /dev/null +++ b/pkg/mentix/entity/entity.go @@ -0,0 +1,28 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package entity + +// Entity is the base interface for all Mentix entities. +type Entity interface { + // GetID returns the ID of the entity. + GetID() string + + // GetName returns the display name of the entity. + GetName() string +} diff --git a/pkg/mentix/entity/registry.go b/pkg/mentix/entity/registry.go new file mode 100644 index 0000000000..0cf44800a9 --- /dev/null +++ b/pkg/mentix/entity/registry.go @@ -0,0 +1,57 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package entity + +import "fmt" + +// Registry represents a simple id->entity map. +type Registry struct { + Entities map[string]Entity +} + +// Register registers a new entity. +func (r *Registry) Register(entity Entity) { + r.Entities[entity.GetID()] = entity +} + +// FindEntities returns all entities matching the provided IDs. +// If an entity with a certain ID doesn't exist and mustExist is true, an error is returned. +func (r *Registry) FindEntities(ids []string, mustExist bool, anyRequired bool) ([]Entity, error) { + var entities []Entity + for _, id := range ids { + if entity, ok := r.Entities[id]; ok { + entities = append(entities, entity) + } else if mustExist { + return nil, fmt.Errorf("no entity with ID '%v' registered", id) + } + } + + if anyRequired && len(entities) == 0 { // At least one entity must be configured + return nil, fmt.Errorf("no entities available") + } + + return entities, nil +} + +// NewRegistry returns a new entity registry. +func NewRegistry() *Registry { + return &Registry{ + Entities: make(map[string]Entity), + } +} diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchange/exchanger.go index 07eb99ef24..abb61038e5 100644 --- a/pkg/mentix/exchange/exchanger.go +++ b/pkg/mentix/exchange/exchanger.go @@ -26,19 +26,19 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" ) // Exchanger is the base interface for importers and exporters. type Exchanger interface { + entity.Entity + // Activate activates the exchanger. Activate(conf *config.Configuration, log *zerolog.Logger) error // Start starts the exchanger; only exchangers which perform periodical background tasks should do something here. Start() error // Stop stops any running background activities of the exchanger. Stop() - - // GetName returns the display name of the exchanger. - GetName() string } // BaseExchanger implements basic exchanger functionality common to all exchangers. diff --git a/pkg/mentix/exchange/exporters/cs3api.go b/pkg/mentix/exchange/exporters/cs3api.go index d3674f0dfc..8a6ff1a435 100755 --- a/pkg/mentix/exchange/exporters/cs3api.go +++ b/pkg/mentix/exchange/exporters/cs3api.go @@ -45,11 +45,16 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo return nil } +// GetID returns the ID of the exporter. +func (exporter *CS3APIExporter) GetID() string { + return config.ExporterIDCS3API +} + // GetName returns the display name of the exporter. func (exporter *CS3APIExporter) GetName() string { return "CS3API" } func init() { - registerExporter(config.ExporterIDCS3API, &CS3APIExporter{}) + registerExporter(&CS3APIExporter{}) } diff --git a/pkg/mentix/exchange/exporters/exporters.go b/pkg/mentix/exchange/exporters/exporters.go index 0a84abe945..19c3deb33d 100644 --- a/pkg/mentix/exchange/exporters/exporters.go +++ b/pkg/mentix/exchange/exporters/exporters.go @@ -19,26 +19,24 @@ package exporters import ( - "fmt" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" "github.com/cs3org/reva/pkg/mentix/exchange" - "github.com/cs3org/reva/pkg/mentix/util/registry" ) // Exporters is a vector of Exporter type Exporters = []Exporter var ( - registeredExporters = registry.NewRegistry() + registeredExporters = entity.NewRegistry() ) // AvailableExporters returns a list of all exporters that are enabled in the configuration. func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { // Try to add all exporters configured in the environment - entries, err := registeredExporters.EntriesByID(conf.EnabledExporters) + entries, err := registeredExporters.FindEntities(conf.EnabledExporters, true, false) if err != nil { return nil, err } @@ -48,11 +46,6 @@ func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { exporters = append(exporters, entry.(Exporter)) } - // At least one exporter must be configured - if len(exporters) == 0 { - return nil, fmt.Errorf("no exporters available") - } - return exporters, nil } @@ -84,6 +77,6 @@ func GetRequestExporters(exporters []Exporter) []exchange.RequestExchanger { return exchange.GetRequestExchangers(asExchangers(exporters)) } -func registerExporter(id string, exporter Exporter) { - registeredExporters.Register(id, exporter) +func registerExporter(exporter Exporter) { + registeredExporters.Register(exporter) } diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchange/exporters/promsd.go index 4179746fc5..f21f4431cf 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchange/exporters/promsd.go @@ -207,11 +207,16 @@ func (exporter *PrometheusSDExporter) exportScrapeConfig(outputFilename string, return nil } +// GetID returns the ID of the exporter. +func (exporter *PrometheusSDExporter) GetID() string { + return config.ExporterIDPrometheusSD +} + // GetName returns the display name of the exporter. func (exporter *PrometheusSDExporter) GetName() string { return "Prometheus SD" } func init() { - registerExporter(config.ExporterIDPrometheusSD, &PrometheusSDExporter{}) + registerExporter(&PrometheusSDExporter{}) } diff --git a/pkg/mentix/exchange/exporters/sitelocations.go b/pkg/mentix/exchange/exporters/sitelocations.go index 21f31cdc7e..f8d1b2525f 100755 --- a/pkg/mentix/exchange/exporters/sitelocations.go +++ b/pkg/mentix/exchange/exporters/sitelocations.go @@ -45,11 +45,16 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log return nil } +// GetID returns the ID of the exporter. +func (exporter *SiteLocationsExporter) GetID() string { + return config.ExporterIDSiteLocations +} + // GetName returns the display name of the exporter. func (exporter *SiteLocationsExporter) GetName() string { return "Site Locations" } func init() { - registerExporter(config.ExporterIDSiteLocations, &SiteLocationsExporter{}) + registerExporter(&SiteLocationsExporter{}) } diff --git a/pkg/mentix/exchange/exporters/webapi.go b/pkg/mentix/exchange/exporters/webapi.go index e981c15456..fbb3ca92a2 100755 --- a/pkg/mentix/exchange/exporters/webapi.go +++ b/pkg/mentix/exchange/exporters/webapi.go @@ -45,11 +45,16 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo return nil } +// GetID returns the ID of the exporter. +func (exporter *WebAPIExporter) GetID() string { + return config.ExporterIDWebAPI +} + // GetName returns the display name of the exporter. func (exporter *WebAPIExporter) GetName() string { return "WebAPI" } func init() { - registerExporter(config.ExporterIDWebAPI, &WebAPIExporter{}) + registerExporter(&WebAPIExporter{}) } diff --git a/pkg/mentix/exchange/importers/importers.go b/pkg/mentix/exchange/importers/importers.go index d76425b345..9385120b2b 100644 --- a/pkg/mentix/exchange/importers/importers.go +++ b/pkg/mentix/exchange/importers/importers.go @@ -22,24 +22,24 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" "github.com/cs3org/reva/pkg/mentix/exchange" - "github.com/cs3org/reva/pkg/mentix/util/registry" ) var ( - registeredImporters = registry.NewRegistry() + registeredImporters = entity.NewRegistry() ) // AvailableImporters returns a list of all importers that are enabled in the configuration. func AvailableImporters(conf *config.Configuration) ([]Importer, error) { // Try to add all importers configured in the environment - entries, err := registeredImporters.EntriesByID(conf.EnabledImporters) + entities, err := registeredImporters.FindEntities(conf.EnabledImporters, true, false) if err != nil { return nil, err } - importers := make([]Importer, 0, len(entries)) - for _, entry := range entries { + importers := make([]Importer, 0, len(entities)) + for _, entry := range entities { importers = append(importers, entry.(Importer)) } @@ -74,6 +74,6 @@ func asExchangers(importers []Importer) []exchange.Exchanger { return exchangers } -func registerImporter(id string, importer Importer) { - registeredImporters.Register(id, importer) +func registerImporter(importer Importer) { + registeredImporters.Register(importer) } diff --git a/pkg/mentix/exchange/importers/webapi.go b/pkg/mentix/exchange/importers/webapi.go index 15208a5595..5bd3e4cebe 100755 --- a/pkg/mentix/exchange/importers/webapi.go +++ b/pkg/mentix/exchange/importers/webapi.go @@ -46,11 +46,16 @@ func (exporter *WebAPIImporter) Activate(conf *config.Configuration, log *zerolo return nil } +// GetID returns the ID of the importer. +func (exporter *WebAPIImporter) GetID() string { + return config.ImporterIDWebAPI +} + // GetName returns the display name of the importer. func (exporter *WebAPIImporter) GetName() string { return "WebAPI" } func init() { - registerImporter(config.ImporterIDWebAPI, &WebAPIImporter{}) + registerImporter(&WebAPIImporter{}) } From 2e174bf141e561a981b6195ba1ae8b4faeae8feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 5 Nov 2020 13:06:28 +0100 Subject: [PATCH 27/35] Unify entities via new Entity types --- pkg/mentix/connectors/connectors.go | 35 +++++++- pkg/mentix/entity/entities.go | 45 ++++++++++ pkg/mentix/exchange/importers/importers.go | 79 ---------------- .../{exchange => exchangers}/exchanger.go | 2 +- .../{exchange => exchangers}/exchangers.go | 44 +++++++-- .../exporters/cs3api.go | 2 +- .../exporters/cs3api/query.go | 0 .../exporters/exporter.go | 6 +- .../exporters/exporters.go | 72 ++++++++------- .../exporters/prometheus/types.go | 0 .../exporters/promsd.go | 2 +- .../exporters/reqexporter.go | 4 +- .../exporters/siteloc/query.go | 0 .../exporters/siteloc/types.go | 0 .../exporters/sitelocations.go | 2 +- .../exporters/webapi.go | 2 +- .../exporters/webapi/query.go | 0 .../importers/importer.go | 12 +-- pkg/mentix/exchangers/importers/importers.go | 90 +++++++++++++++++++ .../importers/reqimporter.go | 4 +- .../importers/webapi.go | 2 +- .../importers/webapi/query.go | 0 .../{exchange => exchangers}/reqexchanger.go | 2 +- pkg/mentix/mentix.go | 71 +++++++-------- 24 files changed, 294 insertions(+), 182 deletions(-) create mode 100644 pkg/mentix/entity/entities.go delete mode 100644 pkg/mentix/exchange/importers/importers.go rename pkg/mentix/{exchange => exchangers}/exchanger.go (99%) rename pkg/mentix/{exchange => exchangers}/exchangers.go (57%) rename pkg/mentix/{exchange => exchangers}/exporters/cs3api.go (96%) rename pkg/mentix/{exchange => exchangers}/exporters/cs3api/query.go (100%) rename pkg/mentix/{exchange => exchangers}/exporters/exporter.go (96%) rename pkg/mentix/{exchange => exchangers}/exporters/exporters.go (54%) rename pkg/mentix/{exchange => exchangers}/exporters/prometheus/types.go (100%) rename pkg/mentix/{exchange => exchangers}/exporters/promsd.go (99%) rename pkg/mentix/{exchange => exchangers}/exporters/reqexporter.go (96%) rename pkg/mentix/{exchange => exchangers}/exporters/siteloc/query.go (100%) rename pkg/mentix/{exchange => exchangers}/exporters/siteloc/types.go (100%) rename pkg/mentix/{exchange => exchangers}/exporters/sitelocations.go (96%) rename pkg/mentix/{exchange => exchangers}/exporters/webapi.go (96%) rename pkg/mentix/{exchange => exchangers}/exporters/webapi/query.go (100%) rename pkg/mentix/{exchange => exchangers}/importers/importer.go (90%) create mode 100644 pkg/mentix/exchangers/importers/importers.go rename pkg/mentix/{exchange => exchangers}/importers/reqimporter.go (97%) rename pkg/mentix/{exchange => exchangers}/importers/webapi.go (96%) rename pkg/mentix/{exchange => exchangers}/importers/webapi/query.go (100%) rename pkg/mentix/{exchange => exchangers}/reqexchanger.go (99%) diff --git a/pkg/mentix/connectors/connectors.go b/pkg/mentix/connectors/connectors.go index 3ba5e5b159..fb48132de3 100644 --- a/pkg/mentix/connectors/connectors.go +++ b/pkg/mentix/connectors/connectors.go @@ -19,6 +19,10 @@ package connectors import ( + "fmt" + + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/entity" ) @@ -27,8 +31,33 @@ var ( registeredConnectors = entity.NewRegistry() ) -// AvailableConnectors returns a list of all connectors that are enabled in the configuration. -func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { +// Collection represents a collection of connectors. +type Collection struct { + Connectors []Connector +} + +// Entities gets the entities in this collection. +func (collection *Collection) Entities() []entity.Entity { + entities := make([]entity.Entity, 0, len(collection.Connectors)) + for _, connector := range collection.Connectors { + entities = append(entities, connector) + } + return entities +} + +func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { + // Activate all connectors + for _, connector := range collection.Connectors { + if err := connector.Activate(conf, log); err != nil { + return fmt.Errorf("unable to activate connector '%v': %v", connector.GetName(), err) + } + } + + return nil +} + +// AvailableConnectors returns a collection of all connectors that are enabled in the configuration. +func AvailableConnectors(conf *config.Configuration) (*Collection, error) { entities, err := registeredConnectors.FindEntities(conf.EnabledConnectors, true, true) if err != nil { return nil, err @@ -39,7 +68,7 @@ func AvailableConnectors(conf *config.Configuration) ([]Connector, error) { connectors = append(connectors, entry.(Connector)) } - return connectors, nil + return &Collection{Connectors: connectors}, nil } func registerConnector(connector Connector) { diff --git a/pkg/mentix/entity/entities.go b/pkg/mentix/entity/entities.go new file mode 100644 index 0000000000..2d6474917d --- /dev/null +++ b/pkg/mentix/entity/entities.go @@ -0,0 +1,45 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package entity + +// Collection is an interface for entity collections. +type Collection interface { + // Entities returns a vector of entities within the collection. + Entities() []Entity +} + +// GetIDs gets a list of entity IDs. +func GetIDs(collection Collection) []string { + entities := collection.Entities() + ids := make([]string, 0, len(entities)) + for _, entity := range entities { + ids = append(ids, entity.GetID()) + } + return ids +} + +// GetNames gets a list of entity names. +func GetNames(collection Collection) []string { + entities := collection.Entities() + names := make([]string, 0, len(entities)) + for _, entity := range entities { + names = append(names, entity.GetName()) + } + return names +} diff --git a/pkg/mentix/exchange/importers/importers.go b/pkg/mentix/exchange/importers/importers.go deleted file mode 100644 index 9385120b2b..0000000000 --- a/pkg/mentix/exchange/importers/importers.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package importers - -import ( - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/entity" - "github.com/cs3org/reva/pkg/mentix/exchange" -) - -var ( - registeredImporters = entity.NewRegistry() -) - -// AvailableImporters returns a list of all importers that are enabled in the configuration. -func AvailableImporters(conf *config.Configuration) ([]Importer, error) { - // Try to add all importers configured in the environment - entities, err := registeredImporters.FindEntities(conf.EnabledImporters, true, false) - if err != nil { - return nil, err - } - - importers := make([]Importer, 0, len(entities)) - for _, entry := range entities { - importers = append(importers, entry.(Importer)) - } - - return importers, nil -} - -// ActivateImporters activates the given importers. -func ActivateImporters(importers []Importer, conf *config.Configuration, log *zerolog.Logger) error { - return exchange.ActivateExchangers(asExchangers(importers), conf, log) -} - -// StartImporters starts the given importers. -func StartImporters(importers []Importer) error { - return exchange.StartExchangers(asExchangers(importers)) -} - -// StopImporters stops the given importers. -func StopImporters(importers []Importer) { - exchange.StopExchangers(asExchangers(importers)) -} - -// GetRequestImporters returns all Importers that implement the RequestExchanger interface. -func GetRequestImporters(importers []Importer) []exchange.RequestExchanger { - return exchange.GetRequestExchangers(asExchangers(importers)) -} - -func asExchangers(importers []Importer) []exchange.Exchanger { - exchangers := make([]exchange.Exchanger, 0, len(importers)) - for _, imp := range importers { - exchangers = append(exchangers, imp) - } - return exchangers -} - -func registerImporter(importer Importer) { - registeredImporters.Register(importer) -} diff --git a/pkg/mentix/exchange/exchanger.go b/pkg/mentix/exchangers/exchanger.go similarity index 99% rename from pkg/mentix/exchange/exchanger.go rename to pkg/mentix/exchangers/exchanger.go index abb61038e5..6e0a79c161 100644 --- a/pkg/mentix/exchange/exchanger.go +++ b/pkg/mentix/exchangers/exchanger.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package exchange +package exchangers import ( "fmt" diff --git a/pkg/mentix/exchange/exchangers.go b/pkg/mentix/exchangers/exchangers.go similarity index 57% rename from pkg/mentix/exchange/exchangers.go rename to pkg/mentix/exchangers/exchangers.go index 0c2d10ef6d..50b92f8872 100644 --- a/pkg/mentix/exchange/exchangers.go +++ b/pkg/mentix/exchangers/exchangers.go @@ -16,19 +16,45 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package exchange +package exchangers import ( "fmt" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" "github.com/rs/zerolog" ) +// Collection is an interface for exchanger collections. +type Collection interface { + entity.Collection + + // Exchangers returns a vector of exchangers within the collection. + Exchangers() []Exchanger +} + +type entityCollectionWrapper struct { + entities []entity.Entity +} + +func (collection *entityCollectionWrapper) Entities() []entity.Entity { + return collection.entities +} + +// AsEntityCollection transforms an exchanger collection into an entity collection. +func AsEntityCollection(collection Collection) entity.Collection { + wrapper := entityCollectionWrapper{} + for _, exchanger := range collection.Exchangers() { + wrapper.entities = append(wrapper.entities, exchanger) + } + return &wrapper +} + // ActivateExchangers activates the given exchangers. -func ActivateExchangers(exchangers []Exchanger, conf *config.Configuration, log *zerolog.Logger) error { - for _, exchanger := range exchangers { +func ActivateExchangers(collection Collection, conf *config.Configuration, log *zerolog.Logger) error { + for _, exchanger := range collection.Exchangers() { if err := exchanger.Activate(conf, log); err != nil { return fmt.Errorf("unable to activate exchanger '%v': %v", exchanger.GetName(), err) } @@ -38,8 +64,8 @@ func ActivateExchangers(exchangers []Exchanger, conf *config.Configuration, log } // StartExchangers starts the given exchangers. -func StartExchangers(exchangers []Exchanger) error { - for _, exchanger := range exchangers { +func StartExchangers(collection Collection) error { + for _, exchanger := range collection.Exchangers() { if err := exchanger.Start(); err != nil { return fmt.Errorf("unable to start exchanger '%v': %v", exchanger.GetName(), err) } @@ -49,16 +75,16 @@ func StartExchangers(exchangers []Exchanger) error { } // StopExchangers stops the given exchangers. -func StopExchangers(exchangers []Exchanger) { - for _, exchanger := range exchangers { +func StopExchangers(collection Collection) { + for _, exchanger := range collection.Exchangers() { exchanger.Stop() } } // GetRequestExchangers gets all exchangers from a vector that implement the RequestExchanger interface. -func GetRequestExchangers(exchangers []Exchanger) []RequestExchanger { +func GetRequestExchangers(collection Collection) []RequestExchanger { var reqExchangers []RequestExchanger - for _, exporter := range exchangers { + for _, exporter := range collection.Exchangers() { if reqExchanger, ok := exporter.(RequestExchanger); ok { reqExchangers = append(reqExchangers, reqExchanger) } diff --git a/pkg/mentix/exchange/exporters/cs3api.go b/pkg/mentix/exchangers/exporters/cs3api.go similarity index 96% rename from pkg/mentix/exchange/exporters/cs3api.go rename to pkg/mentix/exchangers/exporters/cs3api.go index 8a6ff1a435..849de25591 100755 --- a/pkg/mentix/exchange/exporters/cs3api.go +++ b/pkg/mentix/exchangers/exporters/cs3api.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange/exporters/cs3api" + "github.com/cs3org/reva/pkg/mentix/exchangers/exporters/cs3api" ) // CS3APIExporter implements the CS3API exporter. diff --git a/pkg/mentix/exchange/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go similarity index 100% rename from pkg/mentix/exchange/exporters/cs3api/query.go rename to pkg/mentix/exchangers/exporters/cs3api/query.go diff --git a/pkg/mentix/exchange/exporters/exporter.go b/pkg/mentix/exchangers/exporters/exporter.go similarity index 96% rename from pkg/mentix/exchange/exporters/exporter.go rename to pkg/mentix/exchangers/exporters/exporter.go index c0de694c30..a52306d0de 100755 --- a/pkg/mentix/exchange/exporters/exporter.go +++ b/pkg/mentix/exchangers/exporters/exporter.go @@ -21,13 +21,13 @@ package exporters import ( "fmt" - "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Exporter is the interface that all exporters must implement. type Exporter interface { - exchange.Exchanger + exchangers.Exchanger // MeshData returns the mesh data. MeshData() *meshdata.MeshData @@ -38,7 +38,7 @@ type Exporter interface { // BaseExporter implements basic exporter functionality common to all exporters. type BaseExporter struct { - exchange.BaseExchanger + exchangers.BaseExchanger meshData *meshdata.MeshData } diff --git a/pkg/mentix/exchange/exporters/exporters.go b/pkg/mentix/exchangers/exporters/exporters.go similarity index 54% rename from pkg/mentix/exchange/exporters/exporters.go rename to pkg/mentix/exchangers/exporters/exporters.go index 19c3deb33d..d932e82da9 100644 --- a/pkg/mentix/exchange/exporters/exporters.go +++ b/pkg/mentix/exchangers/exporters/exporters.go @@ -23,58 +23,66 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/entity" - "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/exchangers" ) -// Exporters is a vector of Exporter -type Exporters = []Exporter +// Collection represents a collection of exporters. +type Collection struct { + Exporters []Exporter +} var ( registeredExporters = entity.NewRegistry() ) -// AvailableExporters returns a list of all exporters that are enabled in the configuration. -func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { - // Try to add all exporters configured in the environment - entries, err := registeredExporters.FindEntities(conf.EnabledExporters, true, false) - if err != nil { - return nil, err - } +// Entities returns a vector of entities within the collection. +func (collection *Collection) Entities() []entity.Entity { + return exchangers.AsEntityCollection(collection).Entities() +} - exporters := make([]Exporter, 0, len(entries)) - for _, entry := range entries { - exporters = append(exporters, entry.(Exporter)) +// Exchangers returns a vector of exchangers within the collection. +func (collection *Collection) Exchangers() []exchangers.Exchanger { + exchngrs := make([]exchangers.Exchanger, 0, len(collection.Exporters)) + for _, connector := range collection.Exporters { + exchngrs = append(exchngrs, connector) } + return exchngrs +} - return exporters, nil +// ActivateAll activates all exporters. +func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { + return exchangers.ActivateExchangers(collection, conf, log) } -// ActivateExporters activates the given exporters. -func ActivateExporters(exporters []Exporter, conf *config.Configuration, log *zerolog.Logger) error { - return exchange.ActivateExchangers(asExchangers(exporters), conf, log) +// StartAll starts all exporters. +func (collection *Collection) StartAll() error { + return exchangers.StartExchangers(collection) } -// StartExporters starts the given exporters. -func StartExporters(exporters []Exporter) error { - return exchange.StartExchangers(asExchangers(exporters)) +// StopAll stops all exporters. +func (collection *Collection) StopAll() { + exchangers.StopExchangers(collection) } -// StopExporters stops the given exporters. -func StopExporters(exporters []Exporter) { - exchange.StopExchangers(asExchangers(exporters)) +// GetRequestExporters returns all exporters that implement the RequestExchanger interface. +func (collection *Collection) GetRequestExporters() []exchangers.RequestExchanger { + return exchangers.GetRequestExchangers(collection) } -func asExchangers(exporters []Exporter) []exchange.Exchanger { - exchangers := make([]exchange.Exchanger, 0, len(exporters)) - for _, exp := range exporters { - exchangers = append(exchangers, exp) +// AvailableExporters returns a list of all exporters that are enabled in the configuration. +func AvailableExporters(conf *config.Configuration) (*Collection, error) { + // Try to add all exporters configured in the environment + entries, err := registeredExporters.FindEntities(conf.EnabledExporters, true, false) + if err != nil { + return nil, err } - return exchangers -} -// GetRequestExporters returns all exporters that implement the RequestExchanger interface. -func GetRequestExporters(exporters []Exporter) []exchange.RequestExchanger { - return exchange.GetRequestExchangers(asExchangers(exporters)) + exporters := make([]Exporter, 0, len(entries)) + for _, entry := range entries { + exporters = append(exporters, entry.(Exporter)) + } + + return &Collection{Exporters: exporters}, nil } func registerExporter(exporter Exporter) { diff --git a/pkg/mentix/exchange/exporters/prometheus/types.go b/pkg/mentix/exchangers/exporters/prometheus/types.go similarity index 100% rename from pkg/mentix/exchange/exporters/prometheus/types.go rename to pkg/mentix/exchangers/exporters/prometheus/types.go diff --git a/pkg/mentix/exchange/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go similarity index 99% rename from pkg/mentix/exchange/exporters/promsd.go rename to pkg/mentix/exchangers/exporters/promsd.go index f21f4431cf..f73205bc5a 100755 --- a/pkg/mentix/exchange/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -28,7 +28,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange/exporters/prometheus" + "github.com/cs3org/reva/pkg/mentix/exchangers/exporters/prometheus" "github.com/cs3org/reva/pkg/mentix/meshdata" ) diff --git a/pkg/mentix/exchange/exporters/reqexporter.go b/pkg/mentix/exchangers/exporters/reqexporter.go similarity index 96% rename from pkg/mentix/exchange/exporters/reqexporter.go rename to pkg/mentix/exchangers/exporters/reqexporter.go index 4e91e39211..83bca263bf 100644 --- a/pkg/mentix/exchange/exporters/reqexporter.go +++ b/pkg/mentix/exchangers/exporters/reqexporter.go @@ -24,7 +24,7 @@ import ( "net/url" "strings" - "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -37,7 +37,7 @@ type queryCallback func(*meshdata.MeshData, url.Values) (int, []byte, error) // BaseRequestExporter implements basic exporter functionality common to all request exporters. type BaseRequestExporter struct { BaseExporter - exchange.BaseRequestExchanger + exchangers.BaseRequestExchanger defaultActionHandler queryCallback } diff --git a/pkg/mentix/exchange/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go similarity index 100% rename from pkg/mentix/exchange/exporters/siteloc/query.go rename to pkg/mentix/exchangers/exporters/siteloc/query.go diff --git a/pkg/mentix/exchange/exporters/siteloc/types.go b/pkg/mentix/exchangers/exporters/siteloc/types.go similarity index 100% rename from pkg/mentix/exchange/exporters/siteloc/types.go rename to pkg/mentix/exchangers/exporters/siteloc/types.go diff --git a/pkg/mentix/exchange/exporters/sitelocations.go b/pkg/mentix/exchangers/exporters/sitelocations.go similarity index 96% rename from pkg/mentix/exchange/exporters/sitelocations.go rename to pkg/mentix/exchangers/exporters/sitelocations.go index f8d1b2525f..fffd6325c1 100755 --- a/pkg/mentix/exchange/exporters/sitelocations.go +++ b/pkg/mentix/exchangers/exporters/sitelocations.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange/exporters/siteloc" + "github.com/cs3org/reva/pkg/mentix/exchangers/exporters/siteloc" ) // SiteLocationsExporter implements the Site Locations exporter to use with Grafana. diff --git a/pkg/mentix/exchange/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go similarity index 96% rename from pkg/mentix/exchange/exporters/webapi.go rename to pkg/mentix/exchangers/exporters/webapi.go index fbb3ca92a2..346cc10907 100755 --- a/pkg/mentix/exchange/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange/exporters/webapi" + "github.com/cs3org/reva/pkg/mentix/exchangers/exporters/webapi" ) // WebAPIExporter implements the generic Web API exporter. diff --git a/pkg/mentix/exchange/exporters/webapi/query.go b/pkg/mentix/exchangers/exporters/webapi/query.go similarity index 100% rename from pkg/mentix/exchange/exporters/webapi/query.go rename to pkg/mentix/exchangers/exporters/webapi/query.go diff --git a/pkg/mentix/exchange/importers/importer.go b/pkg/mentix/exchangers/importers/importer.go similarity index 90% rename from pkg/mentix/exchange/importers/importer.go rename to pkg/mentix/exchangers/importers/importer.go index 7043d15f74..69ca229986 100755 --- a/pkg/mentix/exchange/importers/importer.go +++ b/pkg/mentix/exchangers/importers/importer.go @@ -23,30 +23,30 @@ import ( "strings" "github.com/cs3org/reva/pkg/mentix/connectors" - "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Importer is the interface that all importers must implement. type Importer interface { - exchange.Exchanger + exchangers.Exchanger // MeshData returns the vector of imported mesh data. MeshData() meshdata.Vector // Process is called periodically to perform the actual import; if data has been imported, true is returned. - Process([]connectors.Connector) (bool, error) + Process(*connectors.Collection) (bool, error) } // BaseImporter implements basic importer functionality common to all importers. type BaseImporter struct { - exchange.BaseExchanger + exchangers.BaseExchanger meshData meshdata.Vector } // Process is called periodically to perform the actual import; if data has been imported, true is returned. -func (importer *BaseImporter) Process(connectors []connectors.Connector) (bool, error) { +func (importer *BaseImporter) Process(connectors *connectors.Collection) (bool, error) { if importer.meshData == nil { // Nothing to do return false, nil } @@ -55,7 +55,7 @@ func (importer *BaseImporter) Process(connectors []connectors.Connector) (bool, // Data is read, so lock it for writing during the loop importer.Locker().RLock() - for _, connector := range connectors { + for _, connector := range connectors.Connectors { if !importer.IsConnectorEnabled(connector.GetID()) { continue } diff --git a/pkg/mentix/exchangers/importers/importers.go b/pkg/mentix/exchangers/importers/importers.go new file mode 100644 index 0000000000..290be3bf6b --- /dev/null +++ b/pkg/mentix/exchangers/importers/importers.go @@ -0,0 +1,90 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/entity" + "github.com/cs3org/reva/pkg/mentix/exchangers" +) + +// Collection represents a collection of importers. +type Collection struct { + Importers []Importer +} + +var ( + registeredImporters = entity.NewRegistry() +) + +// Entities returns a vector of entities within the collection. +func (collection *Collection) Entities() []entity.Entity { + return exchangers.AsEntityCollection(collection).Entities() +} + +// Exchangers returns a vector of exchangers within the collection. +func (collection *Collection) Exchangers() []exchangers.Exchanger { + exchngrs := make([]exchangers.Exchanger, 0, len(collection.Importers)) + for _, connector := range collection.Importers { + exchngrs = append(exchngrs, connector) + } + return exchngrs +} + +// ActivateAll activates all importers. +func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { + return exchangers.ActivateExchangers(collection, conf, log) +} + +// StartAll starts all importers. +func (collection *Collection) StartAll() error { + return exchangers.StartExchangers(collection) +} + +// StopAll stops all importers. +func (collection *Collection) StopAll() { + exchangers.StopExchangers(collection) +} + +// GetRequestImporters returns all importers that implement the RequestExchanger interface. +func (collection *Collection) GetRequestImporters() []exchangers.RequestExchanger { + return exchangers.GetRequestExchangers(collection) +} + +// AvailableImporters returns a collection of all importers that are enabled in the configuration. +func AvailableImporters(conf *config.Configuration) (*Collection, error) { + // Try to add all importers configured in the environment + entities, err := registeredImporters.FindEntities(conf.EnabledImporters, true, false) + if err != nil { + return nil, err + } + + importers := make([]Importer, 0, len(entities)) + for _, entry := range entities { + importers = append(importers, entry.(Importer)) + } + + return &Collection{Importers: importers}, nil +} + +func registerImporter(importer Importer) { + registeredImporters.Register(importer) +} diff --git a/pkg/mentix/exchange/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go similarity index 97% rename from pkg/mentix/exchange/importers/reqimporter.go rename to pkg/mentix/exchangers/importers/reqimporter.go index 3a65327b02..1f4d4767bf 100644 --- a/pkg/mentix/exchange/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -25,7 +25,7 @@ import ( "net/url" "strings" - "github.com/cs3org/reva/pkg/mentix/exchange" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -39,7 +39,7 @@ type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { BaseImporter - exchange.BaseRequestExchanger + exchangers.BaseRequestExchanger registerSiteActionHandler queryCallback unregisterSiteActionHandler queryCallback diff --git a/pkg/mentix/exchange/importers/webapi.go b/pkg/mentix/exchangers/importers/webapi.go similarity index 96% rename from pkg/mentix/exchange/importers/webapi.go rename to pkg/mentix/exchangers/importers/webapi.go index 5bd3e4cebe..ac42fd0750 100755 --- a/pkg/mentix/exchange/importers/webapi.go +++ b/pkg/mentix/exchangers/importers/webapi.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchange/importers/webapi" + "github.com/cs3org/reva/pkg/mentix/exchangers/importers/webapi" ) // WebAPIImporter implements the generic Web API importer. diff --git a/pkg/mentix/exchange/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go similarity index 100% rename from pkg/mentix/exchange/importers/webapi/query.go rename to pkg/mentix/exchangers/importers/webapi/query.go diff --git a/pkg/mentix/exchange/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go similarity index 99% rename from pkg/mentix/exchange/reqexchanger.go rename to pkg/mentix/exchangers/reqexchanger.go index 39446d6d80..c8cd381655 100644 --- a/pkg/mentix/exchange/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package exchange +package exchangers import ( "net/http" diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 981100189e..2a4e476c1b 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -29,9 +29,10 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" - "github.com/cs3org/reva/pkg/mentix/exchange" - "github.com/cs3org/reva/pkg/mentix/exchange/exporters" - "github.com/cs3org/reva/pkg/mentix/exchange/importers" + "github.com/cs3org/reva/pkg/mentix/entity" + "github.com/cs3org/reva/pkg/mentix/exchangers" + "github.com/cs3org/reva/pkg/mentix/exchangers/exporters" + "github.com/cs3org/reva/pkg/mentix/exchangers/importers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -40,9 +41,9 @@ type Mentix struct { conf *config.Configuration log *zerolog.Logger - connectors []connectors.Connector - importers []importers.Importer - exporters []exporters.Exporter + connectors *connectors.Collection + importers *importers.Collection + exporters *exporters.Collection meshDataSet meshdata.Map @@ -86,19 +87,15 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) mntx.meshDataSet = make(meshdata.Map) // Log some infos - connectorNames := make([]string, len(mntx.connectors)) - for idx, connector := range mntx.connectors { - connectorNames[idx] = connector.GetName() - } - importerNames := make([]string, len(mntx.importers)) - for idx, importer := range mntx.importers { - importerNames[idx] = importer.GetName() - } - exporterNames := make([]string, len(mntx.exporters)) - for idx, exporter := range mntx.exporters { - exporterNames[idx] = exporter.GetName() - } - log.Info().Msgf("mentix started with connectors: %v; importers: %v; exporters: %v; update interval: %v", strings.Join(connectorNames, ", "), strings.Join(importerNames, ", "), strings.Join(exporterNames, ", "), duration) + connectorNames := entity.GetNames(mntx.connectors) + importerNames := entity.GetNames(mntx.importers) + exporterNames := entity.GetNames(mntx.exporters) + log.Info().Msgf("mentix started with connectors: %v; importers: %v; exporters: %v; update interval: %v", + strings.Join(connectorNames, ", "), + strings.Join(importerNames, ", "), + strings.Join(exporterNames, ", "), + duration, + ) return nil } @@ -111,11 +108,8 @@ func (mntx *Mentix) initConnectors() error { } mntx.connectors = conns - // Activate all conns - for _, connector := range mntx.connectors { - if err := connector.Activate(mntx.conf, mntx.log); err != nil { - return fmt.Errorf("unable to activate connector '%v': %v", connector.GetName(), err) - } + if err := mntx.connectors.ActivateAll(mntx.conf, mntx.log); err != nil { + return fmt.Errorf("unable to activate connectors: %v", err) } return nil @@ -129,7 +123,7 @@ func (mntx *Mentix) initExchangers() error { } mntx.importers = imps - if err := importers.ActivateImporters(mntx.importers, mntx.conf, mntx.log); err != nil { + if err := mntx.importers.ActivateAll(mntx.conf, mntx.log); err != nil { return fmt.Errorf("unable to activate importers: %v", err) } @@ -140,7 +134,7 @@ func (mntx *Mentix) initExchangers() error { } mntx.exporters = exps - if err := exporters.ActivateExporters(mntx.exporters, mntx.conf, mntx.log); err != nil { + if err := mntx.exporters.ActivateAll(mntx.conf, mntx.log); err != nil { return fmt.Errorf("unable to activate exporters: %v", err) } @@ -149,12 +143,12 @@ func (mntx *Mentix) initExchangers() error { func (mntx *Mentix) startExchangers() error { // Start all importers - if err := importers.StartImporters(mntx.importers); err != nil { + if err := mntx.importers.StartAll(); err != nil { return fmt.Errorf("unable to start importers: %v", err) } // Start all exporters - if err := exporters.StartExporters(mntx.exporters); err != nil { + if err := mntx.exporters.StartAll(); err != nil { return fmt.Errorf("unable to start exporters: %v", err) } @@ -162,12 +156,11 @@ func (mntx *Mentix) startExchangers() error { } func (mntx *Mentix) stopExchangers() { - exporters.StopExporters(mntx.exporters) - importers.StopImporters(mntx.importers) + mntx.exporters.StopAll() + mntx.importers.StopAll() } func (mntx *Mentix) destroy() { - // Stop all im- & exporters mntx.stopExchangers() } @@ -230,7 +223,7 @@ func (mntx *Mentix) tick(updateTimestamp *time.Time) { func (mntx *Mentix) processImporters() (bool, error) { meshDataUpdated := false - for _, importer := range mntx.importers { + for _, importer := range mntx.importers.Importers { updated, err := importer.Process(mntx.connectors) if err != nil { return false, fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err) @@ -248,7 +241,7 @@ func (mntx *Mentix) processImporters() (bool, error) { func (mntx *Mentix) retrieveMeshDataSet() (meshdata.Map, error) { meshDataSet := make(meshdata.Map) - for _, connector := range mntx.connectors { + for _, connector := range mntx.connectors.Connectors { meshData, err := connector.RetrieveMeshData() if err != nil { return nil, fmt.Errorf("retrieving mesh data from connector '%v' failed: %v", connector.GetName(), err) @@ -274,7 +267,7 @@ func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error { mntx.meshDataSet = meshDataSet - for _, exporter := range mntx.exporters { + for _, exporter := range mntx.exporters.Exporters { if err := exporter.Update(meshDataSet); err != nil { return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) } @@ -285,13 +278,13 @@ func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error { } // GetRequestImporters returns all exporters that can handle HTTP requests. -func (mntx *Mentix) GetRequestImporters() []exchange.RequestExchanger { - return importers.GetRequestImporters(mntx.importers) +func (mntx *Mentix) GetRequestImporters() []exchangers.RequestExchanger { + return mntx.importers.GetRequestImporters() } // GetRequestExporters returns all exporters that can handle HTTP requests. -func (mntx *Mentix) GetRequestExporters() []exchange.RequestExchanger { - return exporters.GetRequestExporters(mntx.exporters) +func (mntx *Mentix) GetRequestExporters() []exchangers.RequestExchanger { + return mntx.exporters.GetRequestExporters() } // RequestHandler handles any incoming HTTP requests by asking each RequestExchanger whether it wants to @@ -313,7 +306,7 @@ func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { } } -func (mntx *Mentix) handleRequest(exchangers []exchange.RequestExchanger, w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { +func (mntx *Mentix) handleRequest(exchangers []exchangers.RequestExchanger, w http.ResponseWriter, r *http.Request, log *zerolog.Logger) { // Ask each RequestExchanger if it wants to handle the request for _, exchanger := range exchangers { if exchanger.WantsRequest(r) { From 685db8e3bb132a081cb8e06eabef663b30115413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 5 Nov 2020 13:13:54 +0100 Subject: [PATCH 28/35] Move Activate method into Entity --- pkg/mentix/connectors/connector.go | 2 -- pkg/mentix/connectors/connectors.go | 11 +---------- pkg/mentix/entity/entities.go | 19 +++++++++++++++++++ pkg/mentix/entity/entity.go | 9 +++++++++ pkg/mentix/exchangers/exchanger.go | 2 -- pkg/mentix/exchangers/exchangers.go | 14 -------------- pkg/mentix/exchangers/exporters/exporters.go | 2 +- pkg/mentix/exchangers/importers/importers.go | 2 +- 8 files changed, 31 insertions(+), 30 deletions(-) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 3a5181a049..3f835c455a 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -32,8 +32,6 @@ import ( type Connector interface { entity.Entity - // Activate activates a connector. - Activate(conf *config.Configuration, log *zerolog.Logger) error // RetrieveMeshData fetches new mesh data. RetrieveMeshData() (*meshdata.MeshData, error) // UpdateMeshData updates the provided mesh data on the target side. diff --git a/pkg/mentix/connectors/connectors.go b/pkg/mentix/connectors/connectors.go index fb48132de3..e400167f54 100644 --- a/pkg/mentix/connectors/connectors.go +++ b/pkg/mentix/connectors/connectors.go @@ -19,8 +19,6 @@ package connectors import ( - "fmt" - "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -46,14 +44,7 @@ func (collection *Collection) Entities() []entity.Entity { } func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { - // Activate all connectors - for _, connector := range collection.Connectors { - if err := connector.Activate(conf, log); err != nil { - return fmt.Errorf("unable to activate connector '%v': %v", connector.GetName(), err) - } - } - - return nil + return entity.ActivateEntities(collection, conf, log) } // AvailableConnectors returns a collection of all connectors that are enabled in the configuration. diff --git a/pkg/mentix/entity/entities.go b/pkg/mentix/entity/entities.go index 2d6474917d..a068e04759 100644 --- a/pkg/mentix/entity/entities.go +++ b/pkg/mentix/entity/entities.go @@ -18,12 +18,31 @@ package entity +import ( + "fmt" + + "github.com/cs3org/reva/pkg/mentix/config" + + "github.com/rs/zerolog" +) + // Collection is an interface for entity collections. type Collection interface { // Entities returns a vector of entities within the collection. Entities() []Entity } +// ActivateEntities activates the given entities. +func ActivateEntities(collection Collection, conf *config.Configuration, log *zerolog.Logger) error { + for _, exchanger := range collection.Entities() { + if err := exchanger.Activate(conf, log); err != nil { + return fmt.Errorf("unable to activate entity '%v': %v", exchanger.GetName(), err) + } + } + + return nil +} + // GetIDs gets a list of entity IDs. func GetIDs(collection Collection) []string { entities := collection.Entities() diff --git a/pkg/mentix/entity/entity.go b/pkg/mentix/entity/entity.go index 269dccc643..3d35cf9c9c 100644 --- a/pkg/mentix/entity/entity.go +++ b/pkg/mentix/entity/entity.go @@ -18,11 +18,20 @@ package entity +import ( + "github.com/cs3org/reva/pkg/mentix/config" + + "github.com/rs/zerolog" +) + // Entity is the base interface for all Mentix entities. type Entity interface { // GetID returns the ID of the entity. GetID() string + // Activate activates the entity. + Activate(conf *config.Configuration, log *zerolog.Logger) error + // GetName returns the display name of the entity. GetName() string } diff --git a/pkg/mentix/exchangers/exchanger.go b/pkg/mentix/exchangers/exchanger.go index 6e0a79c161..6fae3ab5c7 100644 --- a/pkg/mentix/exchangers/exchanger.go +++ b/pkg/mentix/exchangers/exchanger.go @@ -33,8 +33,6 @@ import ( type Exchanger interface { entity.Entity - // Activate activates the exchanger. - Activate(conf *config.Configuration, log *zerolog.Logger) error // Start starts the exchanger; only exchangers which perform periodical background tasks should do something here. Start() error // Stop stops any running background activities of the exchanger. diff --git a/pkg/mentix/exchangers/exchangers.go b/pkg/mentix/exchangers/exchangers.go index 50b92f8872..e93d4bb4f1 100644 --- a/pkg/mentix/exchangers/exchangers.go +++ b/pkg/mentix/exchangers/exchangers.go @@ -21,10 +21,7 @@ package exchangers import ( "fmt" - "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/entity" - - "github.com/rs/zerolog" ) // Collection is an interface for exchanger collections. @@ -52,17 +49,6 @@ func AsEntityCollection(collection Collection) entity.Collection { return &wrapper } -// ActivateExchangers activates the given exchangers. -func ActivateExchangers(collection Collection, conf *config.Configuration, log *zerolog.Logger) error { - for _, exchanger := range collection.Exchangers() { - if err := exchanger.Activate(conf, log); err != nil { - return fmt.Errorf("unable to activate exchanger '%v': %v", exchanger.GetName(), err) - } - } - - return nil -} - // StartExchangers starts the given exchangers. func StartExchangers(collection Collection) error { for _, exchanger := range collection.Exchangers() { diff --git a/pkg/mentix/exchangers/exporters/exporters.go b/pkg/mentix/exchangers/exporters/exporters.go index d932e82da9..adee370e4c 100644 --- a/pkg/mentix/exchangers/exporters/exporters.go +++ b/pkg/mentix/exchangers/exporters/exporters.go @@ -51,7 +51,7 @@ func (collection *Collection) Exchangers() []exchangers.Exchanger { // ActivateAll activates all exporters. func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { - return exchangers.ActivateExchangers(collection, conf, log) + return entity.ActivateEntities(collection, conf, log) } // StartAll starts all exporters. diff --git a/pkg/mentix/exchangers/importers/importers.go b/pkg/mentix/exchangers/importers/importers.go index 290be3bf6b..7ce62026e1 100644 --- a/pkg/mentix/exchangers/importers/importers.go +++ b/pkg/mentix/exchangers/importers/importers.go @@ -51,7 +51,7 @@ func (collection *Collection) Exchangers() []exchangers.Exchanger { // ActivateAll activates all importers. func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { - return exchangers.ActivateExchangers(collection, conf, log) + return entity.ActivateEntities(collection, conf, log) } // StartAll starts all importers. From 2a7919f4ab8c2c1741798a2d1e163c9e1852b670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 9 Nov 2020 12:38:28 +0100 Subject: [PATCH 29/35] Update Prometheus target labels --- internal/http/services/mentix/mentix.go | 2 ++ pkg/mentix/exchangers/exporters/promsd.go | 21 +++++++++++---------- pkg/mentix/mentix.go | 2 +- pkg/mentix/meshdata/site.go | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 8c7fff572e..51d8323ef5 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -154,6 +154,8 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Exporters.SiteLocations.Endpoint = "/loc" } addDefaultConnector(&conf.Exporters.SiteLocations.EnabledConnectors) + + addDefaultConnector(&conf.Exporters.PrometheusSD.EnabledConnectors) } // New returns a new Mentix service. diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index f73205bc5a..8606db3238 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -46,11 +46,7 @@ type PrometheusSDExporter struct { } func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { - labels := map[string]string{ - "site": site.Name, - "country": site.CountryCode, - "service_type": endpoint.Type.Name, - } + labels := getScrapeTargetLabels(site, endpoint) // If a metrics path was specified as a property, use that one by setting the corresponding label if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { @@ -70,11 +66,7 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me return nil } - labels := map[string]string{ - "site": site.Name, - "country": site.CountryCode, - "service_type": endpoint.Type.Name, - } + labels := getScrapeTargetLabels(site, endpoint) return &prometheus.ScrapeConfig{ Targets: []string{target}, @@ -82,6 +74,15 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me } } +func getScrapeTargetLabels(site *meshdata.Site, endpoint *meshdata.ServiceEndpoint) map[string]string { + return map[string]string{ + "__meta_site": site.Name, + "__meta_site_type": meshdata.GetSiteTypeName(site.Type), + "__meta_country": site.CountryCode, + "__meta_service_type": endpoint.Type.Name, + } +} + func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Configuration) error { exporter.scrapeCreators = make(map[string]prometheusSDScrapeCreator) diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 2a4e476c1b..88ec87ae3c 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -268,7 +268,7 @@ func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error { mntx.meshDataSet = meshDataSet for _, exporter := range mntx.exporters.Exporters { - if err := exporter.Update(meshDataSet); err != nil { + if err := exporter.Update(mntx.meshDataSet); err != nil { return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) } } diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 92ae753db9..e04d5a0138 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -139,3 +139,17 @@ func (site *Site) GetID() string { return fmt.Sprintf("%s::[%s]", host, site.Name) } + +// GetSiteTypeName returns the readable name of the given site type. +func GetSiteTypeName(siteType SiteType) string { + switch siteType { + case SiteTypeScienceMesh: + return "sciencemesh" + + case SiteTypeCommunity: + return "community" + + default: + return "unknown" + } +} From 3fae7b0a632a1d1db999f670b45b4972cbf3e936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 9 Nov 2020 15:57:23 +0100 Subject: [PATCH 30/35] Expose site ID via Prometheus --- pkg/mentix/exchangers/exporters/promsd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 8606db3238..10ef2f1539 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -78,6 +78,7 @@ func getScrapeTargetLabels(site *meshdata.Site, endpoint *meshdata.ServiceEndpoi return map[string]string{ "__meta_site": site.Name, "__meta_site_type": meshdata.GetSiteTypeName(site.Type), + "__meta_site_id": site.GetID(), "__meta_country": site.CountryCode, "__meta_service_type": endpoint.Type.Name, } From adcf7e615c1441077720d8b7d4d8be29b2ef7dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 9 Nov 2020 16:00:24 +0100 Subject: [PATCH 31/35] Add ..._mentix_... to Prometheus meta labels --- pkg/mentix/exchangers/exporters/promsd.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 10ef2f1539..66d86d269a 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -76,11 +76,11 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me func getScrapeTargetLabels(site *meshdata.Site, endpoint *meshdata.ServiceEndpoint) map[string]string { return map[string]string{ - "__meta_site": site.Name, - "__meta_site_type": meshdata.GetSiteTypeName(site.Type), - "__meta_site_id": site.GetID(), - "__meta_country": site.CountryCode, - "__meta_service_type": endpoint.Type.Name, + "__meta_mentix_site": site.Name, + "__meta_mentix_site_type": meshdata.GetSiteTypeName(site.Type), + "__meta_mentix_site_id": site.GetID(), + "__meta_mentix_country": site.CountryCode, + "__meta_mentix_service_type": endpoint.Type.Name, } } From 64f02cfe3a62ca9cb24a211737610f613eb0c5a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 23 Nov 2020 14:32:26 +0100 Subject: [PATCH 32/35] Cleanup --- pkg/mentix/config/ids.go | 2 ++ pkg/mentix/connectors/connector.go | 6 ++++-- pkg/mentix/entity/entity.go | 5 ++--- pkg/mentix/exchangers/importers/importer.go | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index af655ad984..979c60885f 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,7 +28,9 @@ const ( const ( // ImporterIDWebAPI is the identifier for the WebAPI importer. ImporterIDWebAPI = "webapi" +) +const ( // ExporterIDWebAPI is the identifier for the WebAPI exporter. ExporterIDWebAPI = "webapi" // ExporterIDCS3API is the identifier for the CS3API exporter. diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 3f835c455a..5684ff27eb 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -34,7 +34,8 @@ type Connector interface { // RetrieveMeshData fetches new mesh data. RetrieveMeshData() (*meshdata.MeshData, error) - // UpdateMeshData updates the provided mesh data on the target side. + // UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that + // should be updated, not the entire data set. UpdateMeshData(data *meshdata.MeshData) error } @@ -59,7 +60,8 @@ func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolo return nil } -// UpdateMeshData updates the provided mesh data on the target side. +// UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that +// should be updated, not the entire data set. func (connector *BaseConnector) UpdateMeshData(data *meshdata.MeshData) error { return fmt.Errorf("the connector doesn't support updating of mesh data") } diff --git a/pkg/mentix/entity/entity.go b/pkg/mentix/entity/entity.go index 3d35cf9c9c..11d99f2654 100644 --- a/pkg/mentix/entity/entity.go +++ b/pkg/mentix/entity/entity.go @@ -28,10 +28,9 @@ import ( type Entity interface { // GetID returns the ID of the entity. GetID() string + // GetName returns the display name of the entity. + GetName() string // Activate activates the entity. Activate(conf *config.Configuration, log *zerolog.Logger) error - - // GetName returns the display name of the entity. - GetName() string } diff --git a/pkg/mentix/exchangers/importers/importer.go b/pkg/mentix/exchangers/importers/importer.go index 69ca229986..62413966d6 100755 --- a/pkg/mentix/exchangers/importers/importer.go +++ b/pkg/mentix/exchangers/importers/importer.go @@ -47,7 +47,7 @@ type BaseImporter struct { // Process is called periodically to perform the actual import; if data has been imported, true is returned. func (importer *BaseImporter) Process(connectors *connectors.Collection) (bool, error) { - if importer.meshData == nil { // Nothing to do + if importer.meshData == nil { // No data present for updating, so nothing to process return false, nil } From c804c7540bfa6743929f0d7eeb76de2a953b8033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 23 Nov 2020 15:21:55 +0100 Subject: [PATCH 33/35] Add changelog --- changelog/unreleased/mentix-import.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/mentix-import.md diff --git a/changelog/unreleased/mentix-import.md b/changelog/unreleased/mentix-import.md new file mode 100644 index 0000000000..9979e82d46 --- /dev/null +++ b/changelog/unreleased/mentix-import.md @@ -0,0 +1,5 @@ +Enhancement: Add import support to Mentix + +This update adds import support to Mentix, transforming it into a **Mesh Entity Exchanger**. To properly support vendor site management, a new connector that works on a local file has been added as well. + +https://github.com/cs3org/reva/pull/1332 From a14ad20b0ab2ccacba83d41d98cca62ec55a4cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 23 Nov 2020 15:24:18 +0100 Subject: [PATCH 34/35] Comment fixes --- pkg/mentix/connectors/connectors.go | 1 + pkg/mentix/meshdata/meshdata.go | 2 +- pkg/mentix/meshdata/site.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/mentix/connectors/connectors.go b/pkg/mentix/connectors/connectors.go index e400167f54..d3944e93c5 100644 --- a/pkg/mentix/connectors/connectors.go +++ b/pkg/mentix/connectors/connectors.go @@ -43,6 +43,7 @@ func (collection *Collection) Entities() []entity.Entity { return entities } +// ActivateAll activates all entities in the collection. func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error { return entity.ActivateEntities(collection, conf, log) } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index db00a4191d..5162dab859 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -127,7 +127,7 @@ func (meshData *MeshData) Merge(inData *MeshData) { } } -// Merge removes data from another MeshData instance from this one. +// Unmerge removes data from another MeshData instance from this one. func (meshData *MeshData) Unmerge(inData *MeshData) { for _, site := range inData.Sites { meshData.RemoveSite(site.GetID()) diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index e04d5a0138..7fd5c5d120 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -33,6 +33,7 @@ const ( SiteTypeCommunity ) +// SiteType holds the type of a site. type SiteType int // Site represents a single site managed by Mentix. From c0e65e7ba4873c7317c42473538729f73ccd0d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 23 Nov 2020 15:45:55 +0100 Subject: [PATCH 35/35] Remove all SDK tests --- pkg/sdk/common/common_test.go | 182 --------------------------- pkg/sdk/common/crypto/crypto_test.go | 76 ----------- pkg/sdk/common/net/net_test.go | 159 ----------------------- pkg/sdk/common/testing/testing.go | 62 --------- 4 files changed, 479 deletions(-) delete mode 100644 pkg/sdk/common/common_test.go delete mode 100644 pkg/sdk/common/crypto/crypto_test.go delete mode 100644 pkg/sdk/common/net/net_test.go delete mode 100644 pkg/sdk/common/testing/testing.go diff --git a/pkg/sdk/common/common_test.go b/pkg/sdk/common/common_test.go deleted file mode 100644 index 7736be5332..0000000000 --- a/pkg/sdk/common/common_test.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package common_test - -import ( - "fmt" - "testing" - "time" - - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - - "github.com/cs3org/reva/pkg/sdk/common" - testintl "github.com/cs3org/reva/pkg/sdk/common/testing" -) - -func TestDataDescriptor(t *testing.T) { - const name = "DATA_DESC" - const size = 42 - - dataDesc := common.CreateDataDescriptor(name, size) - now := time.Now().Round(time.Millisecond) - if v := dataDesc.Name(); v != name { - t.Errorf(testintl.FormatTestResult("DataDescriptor.Name", name, v)) - } - if v := dataDesc.Size(); v != size { - t.Errorf(testintl.FormatTestResult("DataDescriptor.Size", size, v)) - } - if v := dataDesc.Mode(); v != 0700 { - t.Errorf(testintl.FormatTestResult("DataDescriptor.Mode", 0700, v)) - } - if v := dataDesc.IsDir(); v != false { - t.Errorf(testintl.FormatTestResult("DataDescriptor.IsDir", false, v)) - } - if v := dataDesc.ModTime(); !v.Round(time.Millisecond).Equal(now) { - // Since there's always a slight chance that the rounded times won't match, just log this mismatch - t.Logf(testintl.FormatTestResult("DataDescriptor.ModTime", now, v)) - } - if v := dataDesc.Sys(); v != nil { - t.Errorf(testintl.FormatTestResult("DataDescriptor.Sys", nil, v)) - } -} - -func TestFindString(t *testing.T) { - tests := []struct { - input []string - needle string - wants int - }{ - {[]string{}, "so empty", -1}, - {[]string{"12345", "hello", "goodbye"}, "hello", 1}, - {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND"}, "Wüste", 2}, - {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND", "Sandiger Grund"}, "Sandiger Grund", 4}, - {[]string{"Nachahmer", "Roger", "k thx bye"}, "thx", -1}, - {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet Under", 0}, - {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet UNDER", -1}, - } - - for _, test := range tests { - found := common.FindString(test.input, test.needle) - if found != test.wants { - t.Errorf(testintl.FormatTestResult("FindString", test.wants, found, test.input, test.needle)) - } - } -} - -func TestFindStringNoCase(t *testing.T) { - tests := []struct { - input []string - needle string - wants int - }{ - {[]string{}, "so empty", -1}, - {[]string{"12345", "hello", "goodbye"}, "hello", 1}, - {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND"}, "Wüste", 2}, - {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND", "Sandiger Grund"}, "Sandiger Grund", 3}, - {[]string{"Nachahmer", "Roger", "k thx bye"}, "thx", -1}, - {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet Under", 0}, - {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet UNDER", 0}, - } - - for _, test := range tests { - found := common.FindStringNoCase(test.input, test.needle) - if found != test.wants { - t.Errorf(testintl.FormatTestResult("FindString", test.wants, found, test.input, test.needle)) - } - } -} - -func TestDecodeOpaqueMap(t *testing.T) { - opaque := types.Opaque{ - Map: map[string]*types.OpaqueEntry{ - "magic": { - Decoder: "plain", - Value: []byte("42"), - }, - "json": { - Decoder: "json", - Value: []byte("[]"), - }, - }, - } - - tests := []struct { - key string - wants string - shouldSucceed bool - }{ - {"magic", "42", true}, - {"json", "[]", false}, - {"somekey", "", false}, - } - - decodedMap := common.DecodeOpaqueMap(&opaque) - for _, test := range tests { - value, ok := decodedMap[test.key] - if ok == test.shouldSucceed { - if ok { - if value != test.wants { - t.Errorf(testintl.FormatTestResult("DecodeOpaqueMap", test.wants, value, opaque)) - } - } - } else { - t.Errorf(testintl.FormatTestResult("DecodeOpaqueMap", test.shouldSucceed, ok, opaque)) - } - } -} - -func TestGetValuesFromOpaque(t *testing.T) { - opaque := types.Opaque{ - Map: map[string]*types.OpaqueEntry{ - "magic": { - Decoder: "plain", - Value: []byte("42"), - }, - "stuff": { - Decoder: "plain", - Value: []byte("Some stuff"), - }, - "json": { - Decoder: "json", - Value: []byte("[]"), - }, - }, - } - - tests := []struct { - keys []string - mandatory bool - shouldSucceed bool - }{ - {[]string{"magic", "stuff"}, true, true}, - {[]string{"magic", "stuff", "json"}, false, true}, - {[]string{"magic", "stuff", "json"}, true, false}, - {[]string{"notfound"}, false, true}, - {[]string{"notfound"}, true, false}, - } - - for _, test := range tests { - _, err := common.GetValuesFromOpaque(&opaque, test.keys, test.mandatory) - if err != nil && test.shouldSucceed { - t.Errorf(testintl.FormatTestError("GetValuesFromOpaque", err, opaque, test.keys, test.mandatory)) - } else if err == nil && !test.shouldSucceed { - t.Errorf(testintl.FormatTestError("GetValuesFromOpaque", fmt.Errorf("getting values from an invalid opaque succeeded"), opaque, test.keys, test.mandatory)) - } - } -} diff --git a/pkg/sdk/common/crypto/crypto_test.go b/pkg/sdk/common/crypto/crypto_test.go deleted file mode 100644 index e5f3c5d91b..0000000000 --- a/pkg/sdk/common/crypto/crypto_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package crypto_test - -import ( - "fmt" - "strings" - "testing" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - - "github.com/cs3org/reva/pkg/sdk/common/crypto" - testintl "github.com/cs3org/reva/pkg/sdk/common/testing" -) - -func TestComputeChecksum(t *testing.T) { - tests := map[string]struct { - checksumType provider.ResourceChecksumType - input string - wants string - }{ - "Unset": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET, "Hello World!", ""}, - "Adler32": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32, "Hello World!", "1c49043e"}, - "SHA1": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_SHA1, "Hello World!", "2ef7bde608ce5404e97d5f042f95f89f1c232871"}, - "MD5": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5, "Hello World!", "ed076287532e86365e841e92bfc50d8c"}, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - if checksum, err := crypto.ComputeChecksum(test.checksumType, strings.NewReader(test.input)); err == nil { - if checksum != test.wants { - t.Errorf(testintl.FormatTestResult("ComputeChecksum", test.wants, checksum, test.checksumType, test.input)) - } - } else { - t.Errorf(testintl.FormatTestError("ComputeChecksum", err)) - } - }) - } - - // Check how ComputeChecksum reacts to an invalid checksum type - if _, err := crypto.ComputeChecksum(provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, nil); err == nil { - t.Errorf(testintl.FormatTestError("ComputeChecksum", fmt.Errorf("accepted an invalid checksum type w/o erring"), provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, nil)) - } -} - -func TestGetChecksumTypeName(t *testing.T) { - tests := map[provider.ResourceChecksumType]string{ - provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET: "unset", - provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_SHA1: "sha1", - provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32: "adler32", - provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5: "md5", - provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID: "invalid", - } - - for input, wants := range tests { - if got := crypto.GetChecksumTypeName(input); got != wants { - t.Errorf(testintl.FormatTestResult("GetChecksumTypeName", wants, got, input)) - } - } -} diff --git a/pkg/sdk/common/net/net_test.go b/pkg/sdk/common/net/net_test.go deleted file mode 100644 index 989a1f093e..0000000000 --- a/pkg/sdk/common/net/net_test.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package net_test - -import ( - "fmt" - "strings" - "testing" - - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - - "github.com/cs3org/reva/pkg/sdk/common" - "github.com/cs3org/reva/pkg/sdk/common/crypto" - "github.com/cs3org/reva/pkg/sdk/common/net" - testintl "github.com/cs3org/reva/pkg/sdk/common/testing" -) - -type rpcStatusTest struct { - status rpc.Code -} - -func (r *rpcStatusTest) GetStatus() *rpc.Status { - return &rpc.Status{ - Code: r.status, - } -} - -func TestCheckRPCInvocation(t *testing.T) { - tests := []struct { - operation string - status rpcStatusTest - shouldSucceed bool - callError error - }{ - {"ok-check", rpcStatusTest{rpc.Code_CODE_OK}, true, nil}, - {"fail-status", rpcStatusTest{rpc.Code_CODE_NOT_FOUND}, false, nil}, - {"fail-err", rpcStatusTest{rpc.Code_CODE_OK}, false, fmt.Errorf("failed")}, - } - - for _, test := range tests { - err := net.CheckRPCInvocation(test.operation, &test.status, test.callError) - if err != nil && test.shouldSucceed { - t.Errorf(testintl.FormatTestError("CheckRPCInvocation", err, test.operation, test.status, test.callError)) - } else if err == nil && !test.shouldSucceed { - t.Errorf(testintl.FormatTestError("CheckRPCInvocation", fmt.Errorf("accepted an invalid RPC invocation"), test.operation, test.status, test.callError)) - } - } -} - -func TestTUSClient(t *testing.T) { - tests := []struct { - endpoint string - shouldSucceed bool - }{ - {"https://tusd.tusdemo.net/files/", true}, - {"https://google.de", false}, - } - - for _, test := range tests { - t.Run(test.endpoint, func(t *testing.T) { - if client, err := net.NewTUSClient(test.endpoint, "", ""); err == nil { - data := strings.NewReader("This is a simple TUS test") - dataDesc := common.CreateDataDescriptor("tus-test.txt", data.Size()) - checksumTypeName := crypto.GetChecksumTypeName(provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5) - - if err := client.Write(data, dataDesc.Name(), &dataDesc, checksumTypeName, ""); err != nil && test.shouldSucceed { - t.Errorf(testintl.FormatTestError("TUSClient.Write", err, data, dataDesc.Name(), &dataDesc, checksumTypeName, "")) - } else if err == nil && !test.shouldSucceed { - t.Errorf(testintl.FormatTestError("TUSClient.Write", fmt.Errorf("writing to a non-TUS host succeeded"), data, dataDesc.Name(), &dataDesc, checksumTypeName, "")) - } - } else { - t.Errorf(testintl.FormatTestError("NewTUSClient", err, test.endpoint, "", "")) - } - }) - } -} - -func TestWebDAVClient(t *testing.T) { - tests := []struct { - endpoint string - shouldSucceed bool - }{ - {"https://zivowncloud2.uni-muenster.de/owncloud/remote.php/dav/files/testUser/", true}, - {"https://google.de", false}, - } - - for _, test := range tests { - t.Run(test.endpoint, func(t *testing.T) { - if client, err := net.NewWebDAVClient(test.endpoint, "testUser", "test12345"); err == nil { - const fileName = "webdav-test.txt" - - data := strings.NewReader("This is a simple WebDAV test") - if err := client.Write(fileName, data, data.Size()); err == nil { - if test.shouldSucceed { - if _, err := client.Read(fileName); err != nil { - t.Errorf(testintl.FormatTestError("WebDAVClient.Read", err)) - } - - if err := client.Remove(fileName); err != nil { - t.Errorf(testintl.FormatTestError("WebDAVClient.Remove", err)) - } - } else { - t.Errorf(testintl.FormatTestError("WebDAVClient.Write", fmt.Errorf("writing to a non-WebDAV host succeeded"), fileName, data, data.Size())) - } - } else if test.shouldSucceed { - t.Errorf(testintl.FormatTestError("WebDAVClient.Write", err, fileName, data, data.Size())) - } - } else { - t.Errorf(testintl.FormatTestError("NewWebDavClient", err, test.endpoint, "testUser", "test12345")) - } - }) - } -} - -func TestHTTPRequest(t *testing.T) { - tests := []struct { - url string - shouldSucceed bool - }{ - {"https://google.de", true}, - {"https://ujhwrgobniwoeo.de", false}, - } - - // Prepare the session - if session, err := testintl.CreateTestSession("sciencemesh-test.uni-muenster.de:9600", "test", "testpass"); err == nil { - for _, test := range tests { - t.Run(test.url, func(t *testing.T) { - if request, err := session.NewHTTPRequest(test.url, "GET", "", nil); err == nil { - if _, err := request.Do(true); err != nil && test.shouldSucceed { - t.Errorf(testintl.FormatTestError("HTTPRequest.Do", err)) - } else if err == nil && !test.shouldSucceed { - t.Errorf(testintl.FormatTestError("HTTPRequest.Do", fmt.Errorf("send request to an invalid host succeeded"))) - } - } else { - t.Errorf(testintl.FormatTestError("Session.NewHTTPRequest", err, test.url, "GET", "", nil)) - } - }) - } - } else { - t.Errorf(testintl.FormatTestError("CreateTestSession", err)) - } -} diff --git a/pkg/sdk/common/testing/testing.go b/pkg/sdk/common/testing/testing.go deleted file mode 100644 index 5a83c45c31..0000000000 --- a/pkg/sdk/common/testing/testing.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package testing - -import ( - "fmt" - "strings" - - "github.com/cs3org/reva/pkg/sdk" -) - -func formatTestMessage(funcName string, msg string, params ...interface{}) string { - // Format parameter list - paramList := make([]string, 0, len(params)) - for _, param := range params { - paramList = append(paramList, fmt.Sprintf("%#v", param)) - } - - return fmt.Sprintf("%s(%s) -> %s", funcName, strings.Join(paramList, ", "), msg) -} - -// FormatTestResult pretty-formats a function call along with its parameters, result and expected result. -func FormatTestResult(funcName string, wants interface{}, got interface{}, params ...interface{}) string { - msg := fmt.Sprintf("Got: %#v; Wants: %#v", got, wants) - return formatTestMessage(funcName, msg, params...) -} - -// FormatTestError pretty-formats a function error. -func FormatTestError(funcName string, err error, params ...interface{}) string { - msg := fmt.Sprintf("Error: %v", err) - return formatTestMessage(funcName, msg, params...) -} - -// CreateTestSession creates a Reva session for testing. -// For this, it performs a basic login using the specified credentials. -func CreateTestSession(host string, username string, password string) (*sdk.Session, error) { - if session, err := sdk.NewSession(); err == nil { - if err := session.Initiate(host, false); err == nil { - if err := session.BasicLogin(username, password); err == nil { - return session, nil - } - } - } - - return nil, fmt.Errorf("unable to create the test session") -}