diff --git a/changelog/unreleased/mesh-metadata-ops.md b/changelog/unreleased/mesh-metadata-ops.md new file mode 100644 index 0000000000..98db51062f --- /dev/null +++ b/changelog/unreleased/mesh-metadata-ops.md @@ -0,0 +1,5 @@ +Enhancement: Mesh meta data operators + +To better support sites that run multiple instances, the meta data have been extended to include a new hierarchy layer called 'operators'. This PR brings all necessary changes in the Mentix and site accounts services. + +https://github.com/cs3org/reva/pull/3072 diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index c71eb414da..1b7a479a23 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -71,30 +71,3 @@ Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data c - **metrics** The [Metrics](metrics) exporter exposes various site-specific metrics through Prometheus. - -## Site Accounts service -Mentix uses the Reva site accounts service to query information about site accounts. The following settings must be configured properly: - -{{% dir name="url" type="string" default="" %}} -The URL of the site accounts service. -{{< highlight toml >}} -[http.services.mentix.accounts] -url = "https://example.com/accounts" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="user" type="string" default="" %}} -The user name to use for basic HTTP authentication. -{{< highlight toml >}} -[http.services.mentix.accounts] -user = "hans" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="password" type="string" default="" %}} -The user password to use for basic HTTP authentication. -{{< highlight toml >}} -[http.services.mentix.accounts] -password = "secret" -{{< /highlight >}} -{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/siteacc/_index.md b/docs/content/en/docs/config/http/services/siteacc/_index.md index a556106c19..4d0473a8db 100644 --- a/docs/content/en/docs/config/http/services/siteacc/_index.md +++ b/docs/content/en/docs/config/http/services/siteacc/_index.md @@ -122,10 +122,10 @@ driver = "file" ### Storage settings - File drivers {{% dir name="sites_file" type="string" default="" %}} -The sites file location. +The operators file location. {{< highlight toml >}} [http.services.siteacc.storage.file] -sites_file = "/var/reva/sites.json" +operators_file = "/var/reva/operators.json" {{< /highlight >}} {{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 340ff4c508..ba410b4d50 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -24,13 +24,6 @@ enabled_connectors = ["gocdb"] # Enable the Metrics exporter [http.services.mentix.exporters.metrics] -# Set up the accounts service used to query information about accounts associated with registered sites -[http.services.mentix.accounts] -# Depending on where the service is running, localhost may also be used here -url = "https://sciencemesh.example.com/iop/accounts" -user = "username" -password = "userpass" - # Configure the Prometheus Service Discovery: [http.services.mentix.exporters.promsd] # The following path must be made available to Prometheus. diff --git a/examples/siteacc/siteacc.toml b/examples/siteacc/siteacc.toml index 68a0c52e4f..128fbf566e 100644 --- a/examples/siteacc/siteacc.toml +++ b/examples/siteacc/siteacc.toml @@ -15,7 +15,7 @@ apikey = "verysecret" [http.services.siteacc.storage] driver = "file" [http.services.siteacc.storage.file] -sites_file = "/var/revad/sites.json" +operators_file = "/var/revad/operators.json" accounts_file = "/var/revad/accounts.json" # Email related settings diff --git a/internal/http/services/siteacc/siteacc.go b/internal/http/services/siteacc/siteacc.go index 16eef4a857..8bed2526ba 100644 --- a/internal/http/services/siteacc/siteacc.go +++ b/internal/http/services/siteacc/siteacc.go @@ -111,10 +111,10 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - // Create the site accounts instance + // Create the sites accounts instance siteacc, err := siteacc.New(conf, log) if err != nil { - return nil, errors.Wrap(err, "error creating the site accounts service") + return nil, errors.Wrap(err, "error creating the sites accounts service") } // Create the service diff --git a/pkg/mentix/accservice/accservice.go b/pkg/mentix/accservice/accservice.go deleted file mode 100644 index 8d53bbba50..0000000000 --- a/pkg/mentix/accservice/accservice.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package accservice - -import ( - "encoding/json" - "fmt" - "net/url" - "path" - "strings" - - "github.com/pkg/errors" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/utils/network" -) - -// RequestResponse holds the response of an accounts service query. -type RequestResponse struct { - Success bool - Error string - Data interface{} -} - -type accountsServiceSettings struct { - URL *url.URL - User string - Password string -} - -var settings accountsServiceSettings - -// Query performs an account service query. -func Query(endpoint string, params network.URLParams) (*RequestResponse, error) { - fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", settings.URL.Scheme, settings.URL.Host), path.Join(settings.URL.Path, endpoint), params) - if err != nil { - return nil, errors.Wrap(err, "error while building the service accounts query URL") - } - - data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: settings.User, Password: settings.Password}, false) - if err != nil { - return nil, errors.Wrap(err, "unable to query the service accounts endpoint") - } - - resp := &RequestResponse{} - if err := json.Unmarshal(data, resp); err != nil { - return nil, errors.Wrap(err, "unable to unmarshal response data") - } - return resp, nil -} - -// GetResponseValue gets a value from an account service query using a dotted path notation. -func GetResponseValue(resp *RequestResponse, path string) interface{} { - if data, ok := resp.Data.(map[string]interface{}); ok { - tokens := strings.Split(path, ".") - for i, name := range tokens { - if i == len(tokens)-1 { - if value, ok := data[name]; ok { - return value - } - } - - if data, ok = data[name].(map[string]interface{}); !ok { - break - } - } - } - - return nil -} - -// InitAccountsService initializes the global accounts service. -func InitAccountsService(conf *config.Configuration) error { - URL, err := url.Parse(conf.AccountsService.URL) - if err != nil { - return errors.Wrap(err, "unable to parse the accounts service URL") - } - - settings.URL = URL - settings.User = conf.AccountsService.User - settings.Password = conf.AccountsService.Password - - return nil -} diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 6ea8abb156..364fb77281 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -66,12 +66,6 @@ type Configuration struct { } `mapstructure:"metrics"` } `mapstructure:"exporters"` - AccountsService struct { - URL string `mapstructure:"url"` - User string `mapstructure:"user"` - Password string `mapstructure:"password"` - } `mapstructure:"accounts"` - // Internal settings EnabledConnectors []string `mapstructure:"-"` EnabledImporters []string `mapstructure:"-"` diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 58b80850b3..43f71556ae 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -66,19 +66,25 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) return nil, fmt.Errorf("could not query service types: %v", err) } - if err := connector.querySites(meshData); err != nil { - return nil, fmt.Errorf("could not query sites: %v", err) + if err := connector.queryNGIs(meshData); err != nil { + return nil, fmt.Errorf("could not query operators: %v", err) } - for _, site := range meshData.Sites { - // Get services associated with the current site - if err := connector.queryServices(meshData, site); err != nil { - return nil, fmt.Errorf("could not query services of site '%v': %v", site.Name, err) + for _, op := range meshData.Operators { + if err := connector.querySites(meshData, op); err != nil { + return nil, fmt.Errorf("could not query sites of operator '%v': %v", op.Name, err) } - // Get downtimes scheduled for the current site - if err := connector.queryDowntimes(meshData, site); err != nil { - return nil, fmt.Errorf("could not query downtimes of site '%v': %v", site.Name, err) + for _, site := range op.Sites { + // Get services associated with the current site + if err := connector.queryServices(meshData, site); err != nil { + return nil, fmt.Errorf("could not query services of site '%v': %v", site.Name, err) + } + + // Get downtimes scheduled for the current site + if err := connector.queryDowntimes(meshData, site); err != nil { + return nil, fmt.Errorf("could not query downtimes of site '%v': %v", site.Name, err) + } } } @@ -124,14 +130,39 @@ func (connector *GOCDBConnector) queryServiceTypes(meshData *meshdata.MeshData) return nil } -func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { +func (connector *GOCDBConnector) queryNGIs(meshData *meshdata.MeshData) error { + var ngis gocdb.NGIs + if err := connector.query(&ngis, "get_ngi", false, true, network.URLParams{}); err != nil { + return err + } + + // Copy retrieved data into the mesh data + meshData.Operators = nil + for _, ngi := range ngis.NGIs { + operator := &meshdata.Operator{ + ID: ngi.Name, + Name: ngi.Name, + Homepage: "", + Email: ngi.Email, + HelpdeskEmail: ngi.HelpdeskEmail, + SecurityEmail: ngi.SecurityEmail, + Sites: nil, + Properties: map[string]string{}, + } + meshData.Operators = append(meshData.Operators, operator) + } + + return nil +} + +func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData, op *meshdata.Operator) error { var sites gocdb.Sites - if err := connector.query(&sites, "get_site", false, true, network.URLParams{}); err != nil { + if err := connector.query(&sites, "get_site", false, true, network.URLParams{"roc": op.ID}); err != nil { return err } // Copy retrieved data into the mesh data - meshData.Sites = nil + op.Sites = nil for _, site := range sites.Sites { properties := connector.extensionsToMap(&site.Extensions) @@ -158,7 +189,7 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { Properties: properties, Downtimes: meshdata.Downtimes{}, } - meshData.Sites = append(meshData.Sites, meshsite) + op.Sites = append(op.Sites, meshsite) } return nil diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index d00d98254a..9227b1322e 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -40,6 +40,19 @@ type ServiceTypes struct { Types []*ServiceType `xml:"SERVICE_TYPE"` } +// NGI represents an NGI in GOCDB. +type NGI struct { + Name string `xml:"NAME"` + Email string `xml:"EMAIL"` + HelpdeskEmail string `xml:"HELPDESK_EMAIL"` + SecurityEmail string `xml:"SECURITY_EMAIL"` +} + +// NGIs is a list of NGI objects. +type NGIs struct { + NGIs []*NGI `xml:"NGI"` +} + // Site represents a site in GOCDB. type Site struct { ShortName string `xml:"SHORT_NAME"` diff --git a/pkg/mentix/exchangers/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go index 2bdc056f72..0c18a39ab3 100755 --- a/pkg/mentix/exchangers/exporters/cs3api/query.go +++ b/pkg/mentix/exchangers/exporters/cs3api/query.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "strings" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" "github.com/cs3org/reva/pkg/mentix/utils" @@ -51,51 +52,54 @@ func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, conf *co func convertMeshDataToOCMData(meshData *meshdata.MeshData, elevatedServiceTypes []string) ([]*ocmprovider.ProviderInfo, error) { // Convert the mesh data into the corresponding OCM data structures - providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Sites)) - for _, site := range meshData.Sites { - // Gather all services from the site - services := make([]*ocmprovider.Service, 0, len(site.Services)) + providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Operators)*3) + for _, op := range meshData.Operators { + for _, site := range op.Sites { + // Gather all services from the site + services := make([]*ocmprovider.Service, 0, len(site.Services)) - addService := func(host string, endpoint *meshdata.ServiceEndpoint, addEndpoints []*ocmprovider.ServiceEndpoint, apiVersion string) { - services = append(services, &ocmprovider.Service{ - Host: host, - Endpoint: convertServiceEndpointToOCMData(endpoint), - AdditionalEndpoints: addEndpoints, - ApiVersion: apiVersion, - }) - } + addService := func(host string, endpoint *meshdata.ServiceEndpoint, addEndpoints []*ocmprovider.ServiceEndpoint, apiVersion string) { + services = append(services, &ocmprovider.Service{ + Host: host, + Endpoint: convertServiceEndpointToOCMData(endpoint), + AdditionalEndpoints: addEndpoints, + ApiVersion: apiVersion, + }) + } - for _, service := range site.Services { - apiVersion := meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, "") + for _, service := range site.Services { + apiVersion := meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, "") - // Gather all additional endpoints of the service - addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints)) - for _, endpoint := range service.AdditionalEndpoints { - if utils.FindInStringArray(endpoint.Type.Name, elevatedServiceTypes, false) != -1 { - endpointURL, _ := url.Parse(endpoint.URL) - addService(endpointURL.Host, endpoint, nil, apiVersion) - } else { - addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint)) + // Gather all additional endpoints of the service + addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints)) + for _, endpoint := range service.AdditionalEndpoints { + if utils.FindInStringArray(endpoint.Type.Name, elevatedServiceTypes, false) != -1 { + endpointURL, _ := url.Parse(endpoint.URL) + addService(endpointURL.Host, endpoint, nil, apiVersion) + } else { + addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint)) + } } + + addService(service.Host, service.ServiceEndpoint, addEndpoints, apiVersion) } - addService(service.Host, service.ServiceEndpoint, addEndpoints, apiVersion) + // Copy the site info into a ProviderInfo + provider := &ocmprovider.ProviderInfo{ + Name: site.Name, + FullName: site.FullName, + Description: site.Description, + Organization: site.Organization, + Domain: site.Domain, + Homepage: site.Homepage, + Email: site.Email, + Services: services, + Properties: site.Properties, + } + provider.Properties[strings.ToUpper(meshdata.PropertyOperator)] = op.ID // Propagate the operator ID as a property + providers = append(providers, provider) } - - // Copy the site info into a ProviderInfo - providers = append(providers, &ocmprovider.ProviderInfo{ - Name: site.Name, - FullName: site.FullName, - Description: site.Description, - Organization: site.Organization, - Domain: site.Domain, - Homepage: site.Homepage, - Email: site.Email, - Services: services, - Properties: site.Properties, - }) } - return providers, nil } diff --git a/pkg/mentix/exchangers/exporters/metrics/metrics.go b/pkg/mentix/exchangers/exporters/metrics/metrics.go index 9863147d07..90a9beb6ca 100644 --- a/pkg/mentix/exchangers/exporters/metrics/metrics.go +++ b/pkg/mentix/exchangers/exporters/metrics/metrics.go @@ -39,6 +39,7 @@ type Metrics struct { } const ( + keyOperatorID = "operator_id" keySiteID = "site_id" keySiteName = "site" keyServiceType = "service_type" @@ -69,7 +70,7 @@ func (m *Metrics) registerMetrics() error { Name: m.isScheduledStats.Name(), Description: m.isScheduledStats.Description(), Measure: m.isScheduledStats, - TagKeys: []tag.Key{tag.MustNewKey(keySiteID), tag.MustNewKey(keySiteName), tag.MustNewKey(keyServiceType)}, + TagKeys: []tag.Key{tag.MustNewKey(keyOperatorID), tag.MustNewKey(keySiteID), tag.MustNewKey(keySiteName), tag.MustNewKey(keyServiceType)}, Aggregation: view.LastValue(), } @@ -82,17 +83,20 @@ func (m *Metrics) registerMetrics() error { // Update is used to update/expose all metrics. func (m *Metrics) Update(meshData *meshdata.MeshData) error { - for _, site := range meshData.Sites { - if err := m.exportSiteMetrics(site); err != nil { - return errors.Wrapf(err, "error while exporting metrics for site '%v'", site.Name) + for _, op := range meshData.Operators { + for _, site := range op.Sites { + if err := m.exportSiteMetrics(site, op); err != nil { + return errors.Wrapf(err, "error while exporting metrics for site '%v'", site.Name) + } } } return nil } -func (m *Metrics) exportSiteMetrics(site *meshdata.Site) error { +func (m *Metrics) exportSiteMetrics(site *meshdata.Site, op *meshdata.Operator) error { mutators := make([]tag.Mutator, 0) + mutators = append(mutators, tag.Insert(tag.MustNewKey(keyOperatorID), op.ID)) mutators = append(mutators, tag.Insert(tag.MustNewKey(keySiteID), site.ID)) mutators = append(mutators, tag.Insert(tag.MustNewKey(keySiteName), site.Name)) mutators = append(mutators, tag.Insert(tag.MustNewKey(keyServiceType), "SCIENCEMESH_HCHECK")) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 2c33c75935..baaf087682 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -36,7 +36,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/meshdata" ) -type prometheusSDScrapeCreatorCallback = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig +type prometheusSDScrapeCreatorCallback = func(op *meshdata.Operator, site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig type prometheusSDScrapeCreator struct { outputFilename string creatorCallback prometheusSDScrapeCreatorCallback @@ -51,41 +51,45 @@ type PrometheusSDExporter struct { } const ( - labelSiteName = "__meta_mentix_site" - labelSiteID = "__meta_mentix_site_id" - labelSiteCountry = "__meta_mentix_site_country" - labelType = "__meta_mentix_type" - labelURL = "__meta_mentix_url" - labelScheme = "__meta_mentix_scheme" - labelHost = "__meta_mentix_host" - labelPort = "__meta_mentix_port" - labelPath = "__meta_mentix_path" - labelServiceHost = "__meta_mentix_service_host" - labelServiceURL = "__meta_mentix_service_url" + labelOperatorName = "__meta_mentix_operator" + labelOperatorID = "__meta_mentix_operator_id" + labelSiteName = "__meta_mentix_site" + labelSiteID = "__meta_mentix_site_id" + labelSiteCountry = "__meta_mentix_site_country" + labelType = "__meta_mentix_type" + labelURL = "__meta_mentix_url" + labelScheme = "__meta_mentix_scheme" + labelHost = "__meta_mentix_host" + labelPort = "__meta_mentix_port" + labelPath = "__meta_mentix_path" + labelServiceHost = "__meta_mentix_service_host" + labelServiceURL = "__meta_mentix_service_url" ) -func createGenericScrapeConfig(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { +func createGenericScrapeConfig(op *meshdata.Operator, site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { endpointURL, _ := url.Parse(endpoint.URL) - labels := getScrapeTargetLabels(site, service, endpoint) + labels := getScrapeTargetLabels(op, site, service, endpoint) return &prometheus.ScrapeConfig{ Targets: []string{endpointURL.Host}, Labels: labels, } } -func getScrapeTargetLabels(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) map[string]string { +func getScrapeTargetLabels(op *meshdata.Operator, site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) map[string]string { endpointURL, _ := url.Parse(endpoint.URL) labels := map[string]string{ - labelSiteName: site.Name, - labelSiteID: site.ID, - labelSiteCountry: site.CountryCode, - labelType: endpoint.Type.Name, - labelURL: endpoint.URL, - labelScheme: endpointURL.Scheme, - labelHost: endpointURL.Hostname(), - labelPort: endpointURL.Port(), - labelPath: endpointURL.Path, - labelServiceHost: service.Host, - labelServiceURL: service.URL, + labelOperatorName: op.Name, + labelOperatorID: op.ID, + labelSiteName: site.Name, + labelSiteID: site.ID, + labelSiteCountry: site.CountryCode, + labelType: endpoint.Type.Name, + labelURL: endpoint.URL, + labelScheme: endpointURL.Scheme, + labelHost: endpointURL.Hostname(), + labelPort: endpointURL.Port(), + labelPath: endpointURL.Path, + labelServiceHost: service.Host, + labelServiceURL: service.URL, } return labels @@ -175,28 +179,30 @@ func (exporter *PrometheusSDExporter) exportMeshData() { func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback prometheusSDScrapeCreatorCallback, serviceFilter []string) []*prometheus.ScrapeConfig { var scrapes []*prometheus.ScrapeConfig - var addScrape = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) { + var addScrape = func(op *meshdata.Operator, site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) { if len(serviceFilter) == 0 || utils.FindInStringArray(endpoint.Type.Name, serviceFilter, false) != -1 { - if scrape := creatorCallback(site, service, endpoint); scrape != nil { + if scrape := creatorCallback(op, site, service, endpoint); scrape != nil { scrapes = append(scrapes, scrape) } } } // Create a scrape config for each service alongside any additional endpoints - for _, site := range exporter.MeshData().Sites { - for _, service := range site.Services { - if !service.IsMonitored { - continue - } + for _, op := range exporter.MeshData().Operators { + for _, site := range op.Sites { + for _, service := range site.Services { + if !service.IsMonitored { + continue + } - // Add the "main" service to the scrapes - addScrape(site, service, service.ServiceEndpoint) + // Add the "main" service to the scrapes + addScrape(op, site, service, service.ServiceEndpoint) - // Add all additional endpoints as well - for _, endpoint := range service.AdditionalEndpoints { - if endpoint.IsMonitored { - addScrape(site, service, endpoint) + // Add all additional endpoints as well + for _, endpoint := range service.AdditionalEndpoints { + if endpoint.IsMonitored { + addScrape(op, site, service, endpoint) + } } } } @@ -205,7 +211,6 @@ func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback promet if scrapes == nil { scrapes = []*prometheus.ScrapeConfig{} } - return scrapes } diff --git a/pkg/mentix/exchangers/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go index ba534e7597..2f478d9fc9 100755 --- a/pkg/mentix/exchangers/exporters/siteloc/query.go +++ b/pkg/mentix/exchangers/exporters/siteloc/query.go @@ -49,14 +49,16 @@ func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *confi func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { // Gather the locations of all sites - locations := make([]*SiteLocation, 0, len(meshData.Sites)) - for _, site := range meshData.Sites { - locations = append(locations, &SiteLocation{ - SiteID: site.ID, - FullName: site.FullName, - Longitude: site.Longitude, - Latitude: site.Latitude, - }) + locations := make([]*SiteLocation, 0, len(meshData.Operators)*3) + for _, op := range meshData.Operators { + for _, site := range op.Sites { + locations = append(locations, &SiteLocation{ + SiteID: site.ID, + FullName: site.FullName, + Longitude: site.Longitude, + Latitude: site.Latitude, + }) + } } return locations, nil diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 16ace2cea7..0b07e811f5 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -27,7 +27,6 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/mentix/accservice" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/entity" @@ -322,11 +321,6 @@ func (mntx *Mentix) handleRequest(exchangers []exchangers.RequestExchanger, w ht // New creates a new Mentix service instance. func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) { - // Configure the accounts service upfront - if err := accservice.InitAccountsService(conf); err != nil { - return nil, fmt.Errorf("unable to initialize the accounts service: %v", err) - } - mntx := new(Mentix) if err := mntx.initialize(conf, log); err != nil { return nil, fmt.Errorf("unable to initialize Mentix: %v", err) diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 7e6ec4e580..69b007c9bb 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -37,7 +37,7 @@ const ( // MeshData holds the entire mesh data managed by Mentix. type MeshData struct { - Sites []*Site + Operators []*Operator ServiceTypes []*ServiceType Status int `json:"-"` @@ -45,39 +45,39 @@ type MeshData struct { // Clear removes all saved data, leaving an empty mesh. func (meshData *MeshData) Clear() { - meshData.Sites = nil + meshData.Operators = nil meshData.ServiceTypes = nil meshData.Status = StatusDefault } -// AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. -func (meshData *MeshData) AddSite(site *Site) { - if siteExisting := meshData.FindSite(site.ID); siteExisting != nil { - *siteExisting = *site +// AddOperator adds a new operator; if an operator with the same ID already exists, the existing one is overwritten. +func (meshData *MeshData) AddOperator(op *Operator) { + if opExisting := meshData.FindOperator(op.ID); opExisting != nil { + *opExisting = *op } else { - meshData.Sites = append(meshData.Sites, site) + meshData.Operators = append(meshData.Operators, op) } } -// RemoveSite removes the provided site. -func (meshData *MeshData) RemoveSite(site *Site) { - for idx, siteExisting := range meshData.Sites { - if strings.EqualFold(siteExisting.ID, site.ID) { // Remove the site by its ID - lastIdx := len(meshData.Sites) - 1 - meshData.Sites[idx] = meshData.Sites[lastIdx] - meshData.Sites[lastIdx] = nil - meshData.Sites = meshData.Sites[:lastIdx] +// RemoveOperator removes the provided operator. +func (meshData *MeshData) RemoveOperator(op *Operator) { + for idx, opExisting := range meshData.Operators { + if strings.EqualFold(opExisting.ID, op.ID) { // Remove the operator by its ID + lastIdx := len(meshData.Operators) - 1 + meshData.Operators[idx] = meshData.Operators[lastIdx] + meshData.Operators[lastIdx] = nil + meshData.Operators = meshData.Operators[:lastIdx] break } } } -// FindSite searches for a site with the given ID. -func (meshData *MeshData) FindSite(id string) *Site { - for _, site := range meshData.Sites { - if strings.EqualFold(site.ID, id) { - return site +// FindOperator searches for an operator with the given ID. +func (meshData *MeshData) FindOperator(id string) *Operator { + for _, op := range meshData.Operators { + if strings.EqualFold(op.ID, id) { + return op } } return nil @@ -117,8 +117,8 @@ func (meshData *MeshData) FindServiceType(name string) *ServiceType { // Merge merges data from another MeshData instance into this one. func (meshData *MeshData) Merge(inData *MeshData) { - for _, site := range inData.Sites { - meshData.AddSite(site) + for _, op := range inData.Operators { + meshData.AddOperator(op) } for _, serviceType := range inData.ServiceTypes { @@ -126,22 +126,11 @@ func (meshData *MeshData) Merge(inData *MeshData) { } } -// Unmerge removes data from another MeshData instance from this one. -func (meshData *MeshData) Unmerge(inData *MeshData) { - for _, site := range inData.Sites { - meshData.RemoveSite(site) - } - - for _, serviceType := range inData.ServiceTypes { - meshData.RemoveServiceType(serviceType) - } -} - // Verify checks if the mesh data is valid. func (meshData *MeshData) Verify() error { - // Verify all sites - for _, site := range meshData.Sites { - if err := site.Verify(); err != nil { + // Verify all operators + for _, op := range meshData.Operators { + if err := op.Verify(); err != nil { return err } } @@ -158,9 +147,9 @@ func (meshData *MeshData) Verify() error { // InferMissingData infers missing data from other data where possible. func (meshData *MeshData) InferMissingData() { - // Infer missing site data - for _, site := range meshData.Sites { - site.InferMissingData() + // Infer missing operator data + for _, op := range meshData.Operators { + op.InferMissingData() } // Infer missing service type data diff --git a/pkg/mentix/meshdata/operator.go b/pkg/mentix/meshdata/operator.go new file mode 100644 index 0000000000..35d73223f9 --- /dev/null +++ b/pkg/mentix/meshdata/operator.go @@ -0,0 +1,110 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +import ( + "fmt" + "strings" +) + +// Operator represents a complete operator including its sites managed by Mentix. +type Operator struct { + ID string + Name string + Homepage string + Email string + HelpdeskEmail string + SecurityEmail string + + Sites []*Site + Properties map[string]string +} + +// AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. +func (op *Operator) AddSite(site *Site) { + if siteExisting := op.FindSite(site.ID); siteExisting != nil { + *siteExisting = *site + } else { + op.Sites = append(op.Sites, site) + } +} + +// RemoveSite removes the provided site. +func (op *Operator) RemoveSite(id string) { + if site := op.FindSite(id); site != nil { + for idx, siteExisting := range op.Sites { + if siteExisting == site { + lastIdx := len(op.Sites) - 1 + op.Sites[idx] = op.Sites[lastIdx] + op.Sites[lastIdx] = nil + op.Sites = op.Sites[:lastIdx] + break + } + } + } +} + +// FindSite searches for a site with the given ID. +func (op *Operator) FindSite(id string) *Site { + for _, site := range op.Sites { + if strings.EqualFold(site.ID, id) { + return site + } + } + return nil +} + +// Verify checks if the operator data is valid. +func (op *Operator) Verify() error { + // Verify data + if op.Name == "" { + return fmt.Errorf("operator name missing") + } + if op.Email == "" { + return fmt.Errorf("operator email missing") + } + + // Verify sites + for _, site := range op.Sites { + if err := site.Verify(); err != nil { + return err + } + } + + return nil +} + +// InferMissingData infers missing data from other data where possible. +func (op *Operator) InferMissingData() { + // Infer missing data + if op.Name == "" { + op.Name = op.ID + } + if op.HelpdeskEmail == "" { + op.HelpdeskEmail = op.Email + } + if op.SecurityEmail == "" { + op.SecurityEmail = op.Email + } + + // Infer missing for sites + for _, site := range op.Sites { + site.InferMissingData() + } +} diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index b4adccb541..e9dd80bf4f 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,6 +21,8 @@ package meshdata import "strings" const ( + // PropertyOperator identifies the operator property. + PropertyOperator = "operator" // PropertySiteID identifies the site ID property. PropertySiteID = "site_id" // PropertyOrganization identifies the organization property. diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index edf45d2ae8..9e6120e2a9 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -28,7 +28,6 @@ import ( // Site represents a single site managed by Mentix. type Site struct { - // Internal settings ID string Name string FullName string @@ -52,7 +51,7 @@ type Site struct { // AddService adds a new service; if a service with the same name already exists, the existing one is overwritten. func (site *Site) AddService(service *Service) { if serviceExisting := site.FindService(service.Name); serviceExisting != nil { - *service = *serviceExisting + *serviceExisting = *service } else { site.Services = append(site.Services, service) } diff --git a/pkg/sdk/action/fileops.go b/pkg/sdk/action/fileops.go index 06a3ad83a8..1e85171ccb 100644 --- a/pkg/sdk/action/fileops.go +++ b/pkg/sdk/action/fileops.go @@ -35,6 +35,16 @@ type FileOperationsAction struct { action } +// GetHome retrieves the home directory path of the current user. +func (action *FileOperationsAction) GetHome() (string, error) { + req := &provider.GetHomeRequest{} + res, err := action.session.Client().GetHome(action.session.Context(), req) + if err := net.CheckRPCInvocation("querying home directory", res, err); err != nil { + return "", err + } + return res.Path, nil +} + // Stat queries the file information of the specified remote resource. func (action *FileOperationsAction) Stat(path string) (*storage.ResourceInfo, error) { ref := &provider.Reference{Path: path} diff --git a/pkg/sdk/action/recycleops.go b/pkg/sdk/action/recycleops.go new file mode 100644 index 0000000000..04ed6fccf7 --- /dev/null +++ b/pkg/sdk/action/recycleops.go @@ -0,0 +1,66 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "fmt" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// RecycleOperationsAction offers recycle bin operations. +type RecycleOperationsAction struct { + action +} + +// Purge purges the entire recycle bin of the current user. +func (action *RecycleOperationsAction) Purge() error { + // Get the home directory to purge the entire recycle bin + fileOpsAct := MustNewFileOperationsAction(action.session) + homePath, err := fileOpsAct.GetHome() + if err != nil { + return err + } + + // Send purge request + ref := &provider.Reference{Path: homePath} + req := &provider.PurgeRecycleRequest{Ref: ref} + res, err := action.session.Client().PurgeRecycle(action.session.Context(), req) + return net.CheckRPCInvocation("purging recycle bin", res, err) +} + +// NewRecycleOperationsAction creates a new recycle operations action. +func NewRecycleOperationsAction(session *sdk.Session) (*RecycleOperationsAction, error) { + action := &RecycleOperationsAction{} + if err := action.initAction(session); err != nil { + return nil, fmt.Errorf("unable to create the RecycleOperationsAction: %v", err) + } + return action, nil +} + +// MustNewRecycleOperationsAction creates a new recycle operations action and panics on failure. +func MustNewRecycleOperationsAction(session *sdk.Session) *RecycleOperationsAction { + action, err := NewRecycleOperationsAction(session) + if err != nil { + panic(err) + } + return action +} diff --git a/pkg/siteacc/account/contact/template.go b/pkg/siteacc/account/contact/template.go index 9a5b76e1be..ff53bf9854 100644 --- a/pkg/siteacc/account/contact/template.go +++ b/pkg/siteacc/account/contact/template.go @@ -79,8 +79,8 @@ const tplBody = `

Contact the ScienceMesh administration using the form below.

Please include as much information as possible in your request, especially:

diff --git a/pkg/siteacc/account/edit/template.go b/pkg/siteacc/account/edit/template.go index 558f0295bd..b82327b76e 100644 --- a/pkg/siteacc/account/edit/template.go +++ b/pkg/siteacc/account/edit/template.go @@ -31,7 +31,7 @@ function verifyForm(formData) { } if (formData.getTrimmed("role") == "") { - setState(STATE_ERROR, "Please specify your role within your site.", "form", "role", true); + setState(STATE_ERROR, "Please specify your role within your operator.", "form", "role", true); return false; } diff --git a/pkg/siteacc/account/manage/template.go b/pkg/siteacc/account/manage/template.go index 5dc597dca7..7eb598b730 100644 --- a/pkg/siteacc/account/manage/template.go +++ b/pkg/siteacc/account/manage/template.go @@ -29,9 +29,9 @@ function handleEditAccount() { window.location.replace("{{getServerAddress}}/account/?path=edit"); } -function handleSiteSettings() { - setState(STATE_STATUS, "Redirecting to the site settings..."); - window.location.replace("{{getServerAddress}}/account/?path=site"); +function handleSitesSettings() { + setState(STATE_STATUS, "Redirecting to the sites settings..."); + window.location.replace("{{getServerAddress}}/account/?path=sites"); } function handleRequestAccess(scope) { @@ -79,7 +79,11 @@ const tplBody = `