From 3f08891ec852113d41a0537f65913d6dbdc174aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 30 Nov 2020 13:49:32 +0100 Subject: [PATCH 01/59] Add a site property to explicitly enable health checks --- pkg/mentix/exchangers/exporters/promsd.go | 6 ++++++ pkg/mentix/meshdata/properties.go | 2 ++ 2 files changed, 8 insertions(+) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 66d86d269a..67065697e2 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "github.com/rs/zerolog" @@ -68,6 +69,11 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me labels := getScrapeTargetLabels(site, endpoint) + // Check if health checks are enabled for the endpoint; if they aren't, skip this endpoint + if enableHealthChecks := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyEnableHealthChecks, "false"); !strings.EqualFold(enableHealthChecks, "true") { + return nil + } + return &prometheus.ScrapeConfig{ Targets: []string{target}, Labels: labels, diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 7dfb8f7a57..479a4631c6 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -25,6 +25,8 @@ const ( PropertyOrganization = "organization" // PropertyMetricsPath identifies the metrics path property. PropertyMetricsPath = "metrics_path" + // PropertyEnableHealthChecks identifies the enable health checks property. + PropertyEnableHealthChecks = "enable_health_checks" // PropertyAPIVersion identifies the API version property. PropertyAPIVersion = "api_version" ) From ed873af3998f5764c78c530154a6cc0c21212bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 30 Nov 2020 13:57:39 +0100 Subject: [PATCH 02/59] Add changelog --- changelog/unreleased/mentix-checks.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/mentix-checks.md diff --git a/changelog/unreleased/mentix-checks.md b/changelog/unreleased/mentix-checks.md new file mode 100644 index 0000000000..928533b283 --- /dev/null +++ b/changelog/unreleased/mentix-checks.md @@ -0,0 +1,5 @@ +Enhancement: Support property to enable health checking on a service + +This update introduces a new service property called `ENABLE_HEALTH_CHECKS` that must be explicitly set to `true` if a service should be checked for its health status. This allows us to only enable these checks for partner sites only, skipping vendor sites. + +https://github.com/cs3org/reva/pull/1347 From 1ffc3773de3a2239eab81028e630db67f6c3967d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 1 Dec 2020 14:11:44 +0100 Subject: [PATCH 03/59] Add some more labels to Prometheus targets --- pkg/mentix/exchangers/exporters/promsd.go | 16 ++++++++++++---- pkg/mentix/meshdata/properties.go | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 67065697e2..2c10c15a35 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -47,7 +47,7 @@ type PrometheusSDExporter struct { } func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { - labels := getScrapeTargetLabels(site, endpoint) + labels := getScrapeTargetLabels(site, host, endpoint) // If a metrics path was specified as a property, use that one by setting the corresponding label if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { @@ -67,7 +67,7 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me return nil } - labels := getScrapeTargetLabels(site, endpoint) + labels := getScrapeTargetLabels(site, host, endpoint) // Check if health checks are enabled for the endpoint; if they aren't, skip this endpoint if enableHealthChecks := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyEnableHealthChecks, "false"); !strings.EqualFold(enableHealthChecks, "true") { @@ -80,14 +80,22 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me } } -func getScrapeTargetLabels(site *meshdata.Site, endpoint *meshdata.ServiceEndpoint) map[string]string { - return map[string]string{ +func getScrapeTargetLabels(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) map[string]string { + labels := map[string]string{ "__meta_mentix_site": site.Name, "__meta_mentix_site_type": meshdata.GetSiteTypeName(site.Type), "__meta_mentix_site_id": site.GetID(), + "__meta_mentix_host": host, "__meta_mentix_country": site.CountryCode, "__meta_mentix_service_type": endpoint.Type.Name, } + + // Get the gRPC port if the corresponding property has been set + if port := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyGRPCPort, ""); len(port) > 0 { + labels["__meta_mentix_grpc_port"] = port + } + + return labels } func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Configuration) error { diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 479a4631c6..b4962b4b0e 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -25,6 +25,8 @@ const ( PropertyOrganization = "organization" // PropertyMetricsPath identifies the metrics path property. PropertyMetricsPath = "metrics_path" + // PropertyGRPCPort identifies the gRPC port property. + PropertyGRPCPort = "grpc_port" // PropertyEnableHealthChecks identifies the enable health checks property. PropertyEnableHealthChecks = "enable_health_checks" // PropertyAPIVersion identifies the API version property. From 83ba2ca59ea3d4ff8266678f70fcd73916576398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 1 Dec 2020 15:46:26 +0100 Subject: [PATCH 04/59] Skip endpoints for BBE if no gRPC port is set --- pkg/mentix/exchangers/exporters/promsd.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 2c10c15a35..6a30cb763d 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -67,13 +67,18 @@ func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *me return nil } - labels := getScrapeTargetLabels(site, host, endpoint) - // Check if health checks are enabled for the endpoint; if they aren't, skip this endpoint if enableHealthChecks := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyEnableHealthChecks, "false"); !strings.EqualFold(enableHealthChecks, "true") { return nil } + labels := getScrapeTargetLabels(site, host, endpoint) + + // For health checks, the gRPC port must be set + if _, ok := labels["__meta_mentix_grpc_port"]; !ok { + return nil + } + return &prometheus.ScrapeConfig{ Targets: []string{target}, Labels: labels, From d67790f82a9699ac911915a9aaef3fadef430182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 4 Jan 2021 16:05:07 +0100 Subject: [PATCH 05/59] Minor TUS adjustments --- pkg/sdk/common/net/tus.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/sdk/common/net/tus.go b/pkg/sdk/common/net/tus.go index 09d96e4892..409a43b77e 100644 --- a/pkg/sdk/common/net/tus.go +++ b/pkg/sdk/common/net/tus.go @@ -52,13 +52,8 @@ func (client *TUSClient) initClient(endpoint string, accessToken string, transpo } client.config.Store = memStore - if accessToken != "" { - client.config.Header.Add(AccessTokenName, accessToken) - } - - if transportToken != "" { - client.config.Header.Add(TransportTokenName, transportToken) - } + client.config.Header.Add(AccessTokenName, accessToken) + client.config.Header.Add(TransportTokenName, transportToken) // Create the TUS client tusClient, err := tus.NewClient(endpoint, client.config) From bf0b0388ea04694bd428d4cb329a6e1de5dc80ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 14 Jan 2021 13:19:55 +0100 Subject: [PATCH 06/59] Set scheme for Prometheus targets --- changelog/unreleased/mentix-prom-scheme.md | 5 +++++ pkg/mentix/exchangers/exporters/promsd.go | 7 +++++++ 2 files changed, 12 insertions(+) create mode 100644 changelog/unreleased/mentix-prom-scheme.md diff --git a/changelog/unreleased/mentix-prom-scheme.md b/changelog/unreleased/mentix-prom-scheme.md new file mode 100644 index 0000000000..8c33586ca0 --- /dev/null +++ b/changelog/unreleased/mentix-prom-scheme.md @@ -0,0 +1,5 @@ +Enhancement: Set scheme for Prometheus targets in Mentix + +This enhancement lets Mentix set the scheme for Prometheus targets. This allows us to also support monitoring of sites that do not support the default HTTPS scheme. + +https://github.com/cs3org/reva/pull/1398 diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 6a30cb763d..e8b4746ea2 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "strings" @@ -49,6 +50,12 @@ type PrometheusSDExporter struct { func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { labels := getScrapeTargetLabels(site, host, endpoint) + // Support both HTTP and HTTPS endpoints by setting the scheme label accordingly + if len(endpoint.URL) > 0 { + if url, err := url.Parse(endpoint.URL); err == nil && (url.Scheme == "http" || url.Scheme == "https") { + labels["__scheme__"] = url.Scheme + } + } // If a metrics path was specified as a property, use that one by setting the corresponding label if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { labels["__metrics_path__"] = metricsPath From 5fe83509e0483986c46428a33e93cab277602dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 13:50:33 +0100 Subject: [PATCH 07/59] Add site authorization status support --- pkg/mentix/connectors/gocdb.go | 3 +++ pkg/mentix/exchangers/exporters/exporter.go | 19 +++++++++++++++++++ .../exchangers/importers/webapi/query.go | 7 +++++++ pkg/mentix/meshdata/properties.go | 2 ++ 4 files changed, 31 insertions(+) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 9ac1a4399c..4fef500127 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -127,6 +127,9 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { for _, site := range sites.Sites { properties := connector.extensionsToMap(&site.Extensions) + // Sites coming from the GOCDB are always authorized + properties[strings.ToUpper(meshdata.PropertyAuthorized)] = "true" + // See if an organization has been defined using properties; otherwise, use the official name organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) diff --git a/pkg/mentix/exchangers/exporters/exporter.go b/pkg/mentix/exchangers/exporters/exporter.go index a52306d0de..6d76b36f64 100755 --- a/pkg/mentix/exchangers/exporters/exporter.go +++ b/pkg/mentix/exchangers/exporters/exporter.go @@ -20,6 +20,7 @@ package exporters import ( "fmt" + "strings" "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" @@ -41,6 +42,8 @@ type BaseExporter struct { exchangers.BaseExchanger meshData *meshdata.MeshData + + allowUnauthorizedSites bool } // Update is called whenever the mesh data set has changed to reflect these changes. @@ -65,6 +68,11 @@ func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.Map) error { if meshDataCloned == nil { return fmt.Errorf("unable to clone the mesh data") } + + if !exporter.allowUnauthorizedSites { + exporter.removeUnauthorizedSites(meshDataCloned) + } + meshDataSetCloned[connectorID] = meshDataCloned } exporter.SetMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) @@ -84,3 +92,14 @@ func (exporter *BaseExporter) SetMeshData(meshData *meshdata.MeshData) { exporter.meshData = meshData } + +func (exporter *BaseExporter) removeUnauthorizedSites(meshData *meshdata.MeshData) { + cleanedSites := make([]*meshdata.Site, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + // Only keep authorized sites + if value := meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false"); strings.EqualFold(value, "true") { + cleanedSites = append(cleanedSites, site) + } + } + meshData.Sites = cleanedSites +} diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index bc99011a80..f4743b2abd 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "github.com/cs3org/reva/pkg/mentix/meshdata" "github.com/cs3org/reva/pkg/mentix/network" @@ -38,6 +39,12 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { if err := meshData.Verify(); err != nil { return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) } + + // Set sites imported through the WebAPI to 'unauthorized' by default + for _, site := range meshData.Sites { + site.Properties[strings.ToUpper(meshdata.PropertyAuthorized)] = "false" + } + meshData.InferMissingData() return meshData, nil } diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index b4962b4b0e..6675b2f997 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,6 +21,8 @@ package meshdata import "strings" const ( + // ProperyAuthorized identifies the authorization status property. + PropertyAuthorized = "authorized" // PropertyOrganization identifies the organization property. PropertyOrganization = "organization" // PropertyMetricsPath identifies the metrics path property. From 7ec1186823781cbe3f0d93e20698b9009b44aa4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 14:22:14 +0100 Subject: [PATCH 08/59] Persist authorization status --- pkg/mentix/connectors/gocdb.go | 6 ++++-- pkg/mentix/connectors/localfile.go | 13 +++++++++++++ pkg/mentix/exchangers/importers/reqimporter.go | 4 +++- pkg/mentix/exchangers/importers/webapi/query.go | 9 +++------ pkg/mentix/meshdata/properties.go | 7 ++++++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 4fef500127..425af0525e 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -127,8 +127,10 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { for _, site := range sites.Sites { properties := connector.extensionsToMap(&site.Extensions) - // Sites coming from the GOCDB are always authorized - properties[strings.ToUpper(meshdata.PropertyAuthorized)] = "true" + // Sites coming from the GOCDB are always authorized by default + if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 { + meshdata.SetPropertyValue(properties, meshdata.PropertyAuthorized, "true") + } // See if an organization has been defined using properties; otherwise, use the official name organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 15918375e3..0a8de0b524 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -93,8 +93,21 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa // Remove data by unmerging meshData.Unmerge(updatedData) } else { + // Store the previous authorization status for already existing sites + siteAuthorizationStatus := make(map[string]string) + for _, site := range meshData.Sites { + siteAuthorizationStatus[site.GetID()] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + } + // Add/update data by merging meshData.Merge(updatedData) + + // Restore the authorization status for all sites + for siteID, status := range siteAuthorizationStatus { + if site := meshData.FindSite(siteID); site != nil { + meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, status) + } + } } // Write the updated sites back to the file diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 1f4d4767bf..f8c49bcc66 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -50,7 +50,9 @@ func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req body, _ := ioutil.ReadAll(req.Body) meshData, status, respData, err := importer.handleQuery(body, req.URL.Query()) if err == nil { - importer.mergeImportedMeshData(meshData) + if len(meshData) > 0 { + importer.mergeImportedMeshData(meshData) + } } else { respData = []byte(err.Error()) } diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index f4743b2abd..b015b141d8 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/meshdata" "github.com/cs3org/reva/pkg/mentix/network" @@ -35,16 +34,14 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { return nil, err } + // Set sites imported through the WebAPI to 'unauthorized' by default + meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} if err := meshData.Verify(); err != nil { return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) } - // Set sites imported through the WebAPI to 'unauthorized' by default - for _, site := range meshData.Sites { - site.Properties[strings.ToUpper(meshdata.PropertyAuthorized)] = "false" - } - meshData.InferMissingData() return meshData, nil } diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 6675b2f997..7339d1796c 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,7 +21,7 @@ package meshdata import "strings" const ( - // ProperyAuthorized identifies the authorization status property. + // PropertyAuthorized identifies the authorization status property. PropertyAuthorized = "authorized" // PropertyOrganization identifies the organization property. PropertyOrganization = "organization" @@ -45,3 +45,8 @@ func GetPropertyValue(props map[string]string, id string, defValue string) strin return defValue } + +// SetPropertyValue sets a property value. +func SetPropertyValue(props map[string]string, id string, value string) { + props[strings.ToUpper(id)] = value +} From 46b2d5335f07f157d533e887bd14b6c8edf660ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 16:48:53 +0100 Subject: [PATCH 09/59] Admin API for site authorization --- internal/http/services/mentix/mentix.go | 28 +++++-- pkg/mentix/config/config.go | 10 +++ pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/localfile.go | 70 +++++++++++++----- pkg/mentix/exchangers/exporters/cs3api.go | 2 +- .../exchangers/exporters/sitelocations.go | 2 +- pkg/mentix/exchangers/exporters/webapi.go | 2 +- pkg/mentix/exchangers/importers/adminapi.go | 60 +++++++++++++++ .../exchangers/importers/adminapi/query.go | 73 +++++++++++++++++++ .../exchangers/importers/reqimporter.go | 11 ++- pkg/mentix/exchangers/importers/webapi.go | 16 ++-- .../exchangers/importers/webapi/query.go | 8 +- pkg/mentix/exchangers/reqexchanger.go | 15 +++- pkg/mentix/meshdata/meshdata.go | 18 +++-- 14 files changed, 265 insertions(+), 52 deletions(-) create mode 100755 pkg/mentix/exchangers/importers/adminapi.go create mode 100755 pkg/mentix/exchangers/importers/adminapi/query.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 51d8323ef5..4eed621baa 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -63,13 +64,20 @@ func (s *svc) Unprotected() []string { importers := s.mntx.GetRequestImporters() exporters := s.mntx.GetRequestExporters() - endpoints := make([]string, len(importers)+len(exporters)) - for idx, importer := range importers { - endpoints[idx] = importer.Endpoint() - } - for idx, exporter := range exporters { - endpoints[idx] = exporter.Endpoint() + getEndpoints := func(exchangers []exchangers.RequestExchanger) []string { + endpoints := make([]string, 0, len(exchangers)) + for _, exchanger := range exchangers { + if !exchanger.IsProtectedEndpoint() { + endpoints = append(endpoints, exchanger.Endpoint()) + } + } + return endpoints } + + endpoints := make([]string, 0, len(importers)+len(exporters)) + endpoints = append(endpoints, getEndpoints(importers)...) + endpoints = append(endpoints, getEndpoints(exporters)...) + return endpoints } @@ -130,7 +138,11 @@ func applyDefaultConfig(conf *config.Configuration) { // Importers if conf.Importers.WebAPI.Endpoint == "" { - conf.Importers.WebAPI.Endpoint = "/" + conf.Importers.WebAPI.Endpoint = "/sites" + } + + if conf.Importers.AdminAPI.Endpoint == "" { + conf.Importers.AdminAPI.Endpoint = "/admin" } // Exporters @@ -141,7 +153,7 @@ func applyDefaultConfig(conf *config.Configuration) { } if conf.Exporters.WebAPI.Endpoint == "" { - conf.Exporters.WebAPI.Endpoint = "/" + conf.Exporters.WebAPI.Endpoint = "/sites" } addDefaultConnector(&conf.Exporters.WebAPI.EnabledConnectors) diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index f43c4e6dfd..7fb38e7ea1 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -38,23 +38,33 @@ type Configuration struct { Importers struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` + + AdminAPI struct { + Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` + } `mapstructure:"adminapi"` } `mapstructure:"importers"` Exporters struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` CS3API struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"cs3api"` SiteLocations struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"siteloc"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 979c60885f..a49c242209 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,6 +28,8 @@ const ( const ( // ImporterIDWebAPI is the identifier for the WebAPI importer. ImporterIDWebAPI = "webapi" + // ImporterIDAdminAPI is the identifier for the AdminAPI importer. + ImporterIDAdminAPI = "adminapi" ) const ( diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 0a8de0b524..c6d5b39b54 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -89,25 +89,19 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa meshData = &meshdata.MeshData{} } - if (updatedData.Flags & meshdata.FlagObsolete) == meshdata.FlagObsolete { - // Remove data by unmerging - meshData.Unmerge(updatedData) - } else { - // Store the previous authorization status for already existing sites - siteAuthorizationStatus := make(map[string]string) - for _, site := range meshData.Sites { - siteAuthorizationStatus[site.GetID()] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") - } + err = nil + switch updatedData.Status { + case meshdata.StatusDefault: + err = connector.mergeData(meshData, updatedData) - // Add/update data by merging - meshData.Merge(updatedData) + case meshdata.StatusObsolete: + err = connector.unmergeData(meshData, updatedData) - // Restore the authorization status for all sites - for siteID, status := range siteAuthorizationStatus { - if site := meshData.FindSite(siteID); site != nil { - meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, status) - } - } + case meshdata.StatusAuthorize: + err = connector.authorizeData(meshData, updatedData, true) + + case meshdata.StatusUnauthorize: + err = connector.authorizeData(meshData, updatedData, false) } // Write the updated sites back to the file @@ -119,6 +113,48 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa return nil } +func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { + // Store the previous authorization status for already existing sites + siteAuthorizationStatus := make(map[string]string) + for _, site := range meshData.Sites { + siteAuthorizationStatus[site.GetID()] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + } + + // Add/update data by merging + meshData.Merge(updatedData) + + // Restore the authorization status for all sites + for siteID, status := range siteAuthorizationStatus { + if site := meshData.FindSite(siteID); site != nil { + meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, status) + } + } + return nil +} + +func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { + // Remove data by unmerging + meshData.Unmerge(updatedData) + return nil +} + +func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData, authorize bool) error { + for _, placeholderSite := range updatedData.Sites { + // The site ID is stored in the updated site's name + if site := meshData.FindSite(placeholderSite.Name); site != nil { + if authorize { + meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "true") + } else { + meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + } + } else { + return fmt.Errorf("no site with id '%v' found", placeholderSite.Name) + } + } + + return nil +} + func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { for _, site := range meshData.Sites { site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites diff --git a/pkg/mentix/exchangers/exporters/cs3api.go b/pkg/mentix/exchangers/exporters/cs3api.go index 849de25591..b224eadc65 100755 --- a/pkg/mentix/exchangers/exporters/cs3api.go +++ b/pkg/mentix/exchangers/exporters/cs3api.go @@ -37,7 +37,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo } // Store CS3API specifics - exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint) + exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint, conf.Exporters.CS3API.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors) exporter.defaultActionHandler = cs3api.HandleDefaultQuery diff --git a/pkg/mentix/exchangers/exporters/sitelocations.go b/pkg/mentix/exchangers/exporters/sitelocations.go index fffd6325c1..33ad4c838c 100755 --- a/pkg/mentix/exchangers/exporters/sitelocations.go +++ b/pkg/mentix/exchangers/exporters/sitelocations.go @@ -37,7 +37,7 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log } // Store SiteLocations specifics - exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint) + exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint, conf.Exporters.SiteLocations.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors) exporter.defaultActionHandler = siteloc.HandleDefaultQuery diff --git a/pkg/mentix/exchangers/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go index 346cc10907..36cdeabed2 100755 --- a/pkg/mentix/exchangers/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -37,7 +37,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo } // Store WebAPI specifics - exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint) + exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint, conf.Exporters.WebAPI.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) exporter.defaultActionHandler = webapi.HandleDefaultQuery diff --git a/pkg/mentix/exchangers/importers/adminapi.go b/pkg/mentix/exchangers/importers/adminapi.go new file mode 100755 index 0000000000..53ee50748b --- /dev/null +++ b/pkg/mentix/exchangers/importers/adminapi.go @@ -0,0 +1,60 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchangers/importers/adminapi" +) + +// AdminAPIImporter implements the administrative API importer. +type AdminAPIImporter struct { + BaseRequestImporter +} + +// Activate activates the importer. +func (importer *AdminAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { + return err + } + + // Store AdminAPI specifics + importer.SetEndpoint(conf.Importers.AdminAPI.Endpoint, conf.Importers.AdminAPI.IsProtected) + importer.SetEnabledConnectors(conf.Importers.AdminAPI.EnabledConnectors) + + importer.authorizeSiteActionHandler = adminapi.HandleAuthorizeSiteQuery + + return nil +} + +// GetID returns the ID of the importer. +func (importer *AdminAPIImporter) GetID() string { + return config.ImporterIDAdminAPI +} + +// GetName returns the display name of the importer. +func (importer *AdminAPIImporter) GetName() string { + return "AdminAPI" +} + +func init() { + registerImporter(&AdminAPIImporter{}) +} diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go new file mode 100755 index 0000000000..f98902c594 --- /dev/null +++ b/pkg/mentix/exchangers/importers/adminapi/query.go @@ -0,0 +1,73 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package adminapi + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/network" +) + +func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { + jsonData := make(map[string]interface{}) + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, err + } + + if value, ok := jsonData["id"]; ok { + if id, ok := value.(string); ok { + site := &meshdata.Site{} + site.Name = id // Store the provided site ID in the site's name + + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + return meshData, nil + } else { + return nil, fmt.Errorf("site id invalid") + } + } else { + return nil, fmt.Errorf("site id missing") + } +} + +func handleAdminQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { + meshData, err := decodeAdminQueryData(data) + if err != nil { + return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil + } + meshData.Status = status + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].Name}), nil +} + +// HandleAuthorizeSiteQuery sets the authorization status of a site. +func HandleAuthorizeSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + status := params.Get("status") + + if strings.EqualFold(status, "true") { + return handleAdminQuery(data, params, meshdata.StatusAuthorize, "SITE_AUTHORIZED") + } else if strings.EqualFold(status, "false") { + return handleAdminQuery(data, params, meshdata.StatusUnauthorize, "SITE_UNAUTHORIZED") + } + + return nil, http.StatusBadRequest, network.CreateResponse("INVALID_QUERY", network.ResponseParams{}), nil +} diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index f8c49bcc66..ae4c6c7d65 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -32,6 +32,7 @@ import ( const ( queryActionRegisterSite = "register" queryActionUnregisterSite = "unregister" + queryActionAuthorizeSite = "authorize" ) type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) @@ -43,12 +44,13 @@ type BaseRequestImporter struct { registerSiteActionHandler queryCallback unregisterSiteActionHandler queryCallback + authorizeSiteActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { body, _ := ioutil.ReadAll(req.Body) - meshData, status, respData, err := importer.handleQuery(body, req.URL.Query()) + meshData, status, respData, err := importer.handleQuery(body, req.URL.Path, req.URL.Query()) if err == nil { if len(meshData) > 0 { importer.mergeImportedMeshData(meshData) @@ -73,7 +75,7 @@ func (importer *BaseRequestImporter) mergeImportedMeshData(meshData meshdata.Vec } } -func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func (importer *BaseRequestImporter) handleQuery(data []byte, path string, params url.Values) (meshdata.Vector, int, []byte, error) { action := params.Get("action") switch strings.ToLower(action) { case queryActionRegisterSite: @@ -86,6 +88,11 @@ func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) return importer.unregisterSiteActionHandler(data, params) } + case queryActionAuthorizeSite: + if importer.authorizeSiteActionHandler != nil { + return importer.authorizeSiteActionHandler(data, params) + } + default: return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) } diff --git a/pkg/mentix/exchangers/importers/webapi.go b/pkg/mentix/exchangers/importers/webapi.go index ac42fd0750..1a542d59ad 100755 --- a/pkg/mentix/exchangers/importers/webapi.go +++ b/pkg/mentix/exchangers/importers/webapi.go @@ -31,28 +31,28 @@ type WebAPIImporter struct { } // Activate activates the importer. -func (exporter *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseRequestImporter.Activate(conf, log); err != nil { +func (importer *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { return err } // Store WebAPI specifics - exporter.SetEndpoint(conf.Importers.WebAPI.Endpoint) - exporter.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) + importer.SetEndpoint(conf.Importers.WebAPI.Endpoint, conf.Importers.WebAPI.IsProtected) + importer.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) - exporter.registerSiteActionHandler = webapi.HandleRegisterSiteQuery - exporter.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery + importer.registerSiteActionHandler = webapi.HandleRegisterSiteQuery + importer.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery return nil } // GetID returns the ID of the importer. -func (exporter *WebAPIImporter) GetID() string { +func (importer *WebAPIImporter) GetID() string { return config.ImporterIDWebAPI } // GetName returns the display name of the importer. -func (exporter *WebAPIImporter) GetName() string { +func (importer *WebAPIImporter) GetName() string { return "WebAPI" } diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index b015b141d8..9929ce7881 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -46,21 +46,21 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { return meshData, nil } -func handleQuery(data []byte, params url.Values, flags int32, msg string) (meshdata.Vector, int, []byte, error) { +func handleQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { meshData, err := decodeQueryData(data) if err != nil { return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil } - meshData.Flags = flags + meshData.Status = status return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].GetID()}), nil } // HandleRegisterSiteQuery registers a site. func HandleRegisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.FlagsNone, "SITE_REGISTERED") + return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. func HandleUnregisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.FlagObsolete, "SITE_UNREGISTERED") + return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/exchangers/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go index c8cd381655..fcbb4daf80 100644 --- a/pkg/mentix/exchangers/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -27,6 +27,8 @@ import ( type RequestExchanger interface { // Endpoint returns the (relative) endpoint of the exchanger. Endpoint() string + // IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization. + IsProtectedEndpoint() bool // WantsRequest returns whether the exchanger wants to handle the incoming request. WantsRequest(r *http.Request) bool // HandleRequest handles the actual HTTP request. @@ -37,7 +39,8 @@ type RequestExchanger interface { type BaseRequestExchanger struct { RequestExchanger - endpoint string + endpoint string + isProtectedEndpoint bool } // Endpoint returns the (relative) endpoint of the exchanger. @@ -50,12 +53,18 @@ func (exchanger *BaseRequestExchanger) Endpoint() string { return strings.TrimSpace(endpoint) } +// IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization. +func (exchanger *BaseRequestExchanger) IsProtectedEndpoint() bool { + return exchanger.isProtectedEndpoint +} + // SetEndpoint sets the (relative) endpoint of the exchanger. -func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string) { +func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string, isProtected bool) { exchanger.endpoint = endpoint + exchanger.isProtectedEndpoint = isProtected } -// WantsRequest returns whether the exporter wants to handle the incoming request. +// WantsRequest returns whether the exchanger wants to handle the incoming request. func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool { return r.URL.Path == exchanger.Endpoint() } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 5162dab859..83f5344a33 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -25,11 +25,15 @@ import ( ) const ( - // FlagsNone resets all mesh data flags. - FlagsNone = 0 - - // FlagObsolete flags the mesh data for removal. - FlagObsolete = 0x0001 + // StatusDefault signals that this is just regular data. + StatusDefault = iota + + // StatusObsolete flags the mesh data for removal. + StatusObsolete + // StatusAuthorize flags the mesh data for authorization. + StatusAuthorize + // StatusUnauthorize flags the mesh data for unauthorization. + StatusUnauthorize ) // MeshData holds the entire mesh data managed by Mentix. @@ -37,7 +41,7 @@ type MeshData struct { Sites []*Site ServiceTypes []*ServiceType - Flags int32 `json:"-"` + Status int `json:"-"` } // Clear removes all saved data, leaving an empty mesh. @@ -45,7 +49,7 @@ func (meshData *MeshData) Clear() { meshData.Sites = nil meshData.ServiceTypes = nil - meshData.Flags = FlagsNone + meshData.Status = StatusDefault } // AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. From c3e082e8f92e388b02c10dc393c7b5bc87bf803c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 16:52:13 +0100 Subject: [PATCH 10/59] Update changelog --- changelog/unreleased/mentix-prom-scheme.md | 5 ----- changelog/unreleased/mentix-site-auth.md | 7 +++++++ 2 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 changelog/unreleased/mentix-prom-scheme.md create mode 100644 changelog/unreleased/mentix-site-auth.md diff --git a/changelog/unreleased/mentix-prom-scheme.md b/changelog/unreleased/mentix-prom-scheme.md deleted file mode 100644 index 8c33586ca0..0000000000 --- a/changelog/unreleased/mentix-prom-scheme.md +++ /dev/null @@ -1,5 +0,0 @@ -Enhancement: Set scheme for Prometheus targets in Mentix - -This enhancement lets Mentix set the scheme for Prometheus targets. This allows us to also support monitoring of sites that do not support the default HTTPS scheme. - -https://github.com/cs3org/reva/pull/1398 diff --git a/changelog/unreleased/mentix-site-auth.md b/changelog/unreleased/mentix-site-auth.md new file mode 100644 index 0000000000..22372c181a --- /dev/null +++ b/changelog/unreleased/mentix-site-auth.md @@ -0,0 +1,7 @@ +Enhancement: Support site authorization status in Mentix + +This enhancement adds support for a site authorization status to Mentix. This way, sites registered via a web app can now be excluded until authorized manually by an administrator. + +Furthermore, Mentix now sets the scheme for Prometheus targets. This allows us to also support monitoring of sites that do not support the default HTTPS scheme. + +https://github.com/cs3org/reva/pull/1398 From 63836d1941812ad94eb04dca5b0fd167db45966d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 16:54:33 +0100 Subject: [PATCH 11/59] Hound fix --- pkg/mentix/exchangers/importers/adminapi/query.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go index f98902c594..ede43f9f27 100755 --- a/pkg/mentix/exchangers/importers/adminapi/query.go +++ b/pkg/mentix/exchangers/importers/adminapi/query.go @@ -45,9 +45,9 @@ func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { } else { return nil, fmt.Errorf("site id invalid") } - } else { - return nil, fmt.Errorf("site id missing") } + + return nil, fmt.Errorf("site id missing") } func handleAdminQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { From c39ee4f9562510b3edaa2bb262cd0b6d3d5bc3a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 16:55:23 +0100 Subject: [PATCH 12/59] Hound fix --- pkg/mentix/exchangers/importers/adminapi/query.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go index ede43f9f27..36c2265849 100755 --- a/pkg/mentix/exchangers/importers/adminapi/query.go +++ b/pkg/mentix/exchangers/importers/adminapi/query.go @@ -42,9 +42,9 @@ func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} return meshData, nil - } else { - return nil, fmt.Errorf("site id invalid") } + + return nil, fmt.Errorf("site id invalid") } return nil, fmt.Errorf("site id missing") From 0fe77e731f56db478dbd16a4a5296e93ee55b144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 Jan 2021 17:11:37 +0100 Subject: [PATCH 13/59] Missing error check --- pkg/mentix/connectors/localfile.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index c6d5b39b54..7642e1897e 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -104,6 +104,10 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa err = connector.authorizeData(meshData, updatedData, false) } + if err != nil { + return err + } + // Write the updated sites back to the file jsonData, _ := json.MarshalIndent(meshData.Sites, "", "\t") if err := ioutil.WriteFile(connector.filePath, jsonData, 0755); err != nil { From 990af828a1e099ec1df6bbc3e89c0b11e2105fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 Jan 2021 15:47:10 +0100 Subject: [PATCH 14/59] Expose site ID --- pkg/mentix/connectors/gocdb.go | 1 + pkg/mentix/connectors/localfile.go | 3 ++- pkg/mentix/exchangers/exporters/promsd.go | 2 +- pkg/mentix/exchangers/exporters/webapi.go | 2 ++ pkg/mentix/exchangers/importers/webapi/query.go | 5 ++++- pkg/mentix/meshdata/meshdata.go | 6 +++--- pkg/mentix/meshdata/site.go | 12 +++++++++--- 7 files changed, 22 insertions(+), 9 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 425af0525e..8b008ec124 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -75,6 +75,7 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) } } + meshData.InferMissingData() return meshData, nil } diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 7642e1897e..1cfeae7ae5 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -78,6 +78,7 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err // Update the site types, as these are not part of the JSON data connector.setSiteTypes(meshData) + meshData.InferMissingData() return meshData, nil } @@ -121,7 +122,7 @@ func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, upda // Store the previous authorization status for already existing sites siteAuthorizationStatus := make(map[string]string) for _, site := range meshData.Sites { - siteAuthorizationStatus[site.GetID()] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + siteAuthorizationStatus[site.ID] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") } // Add/update data by merging diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index e8b4746ea2..0cb1fba4c8 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -96,7 +96,7 @@ func getScrapeTargetLabels(site *meshdata.Site, host string, endpoint *meshdata. labels := map[string]string{ "__meta_mentix_site": site.Name, "__meta_mentix_site_type": meshdata.GetSiteTypeName(site.Type), - "__meta_mentix_site_id": site.GetID(), + "__meta_mentix_site_id": site.ID, "__meta_mentix_host": host, "__meta_mentix_country": site.CountryCode, "__meta_mentix_service_type": endpoint.Type.Name, diff --git a/pkg/mentix/exchangers/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go index 36cdeabed2..30f2b18bd6 100755 --- a/pkg/mentix/exchangers/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -40,6 +40,8 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint, conf.Exporters.WebAPI.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) + exporter.allowUnauthorizedSites = true + exporter.defaultActionHandler = webapi.HandleDefaultQuery return nil diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index 9929ce7881..189bb849ef 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -34,6 +34,9 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { return nil, err } + // Imported sites will be assigned an ID automatically + site.ID = "" + // Set sites imported through the WebAPI to 'unauthorized' by default meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") @@ -52,7 +55,7 @@ func handleQuery(data []byte, params url.Values, status int, msg string) (meshda return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil } meshData.Status = status - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].GetID()}), nil + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].ID}), nil } // HandleRegisterSiteQuery registers a site. diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 83f5344a33..40832c503f 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -54,7 +54,7 @@ func (meshData *MeshData) Clear() { // AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. func (meshData *MeshData) AddSite(site *Site) { - if siteExisting := meshData.FindSite(site.GetID()); siteExisting != nil { + if siteExisting := meshData.FindSite(site.ID); siteExisting != nil { *siteExisting = *site } else { meshData.Sites = append(meshData.Sites, site) @@ -79,7 +79,7 @@ func (meshData *MeshData) RemoveSite(id string) { // FindSite searches for a site with the given ID. func (meshData *MeshData) FindSite(id string) *Site { for _, site := range meshData.Sites { - if strings.EqualFold(site.GetID(), id) { + if strings.EqualFold(site.ID, id) { return site } } @@ -134,7 +134,7 @@ func (meshData *MeshData) Merge(inData *MeshData) { // Unmerge removes data from another MeshData instance from this one. func (meshData *MeshData) Unmerge(inData *MeshData) { for _, site := range inData.Sites { - meshData.RemoveSite(site.GetID()) + meshData.RemoveSite(site.ID) } for _, serviceType := range inData.ServiceTypes { diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 7fd5c5d120..da6ffce2d6 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -40,6 +40,7 @@ type SiteType int type Site struct { Type SiteType `json:"-"` + ID string Name string FullName string Organization string @@ -122,15 +123,20 @@ func (site *Site) InferMissingData() { } } + // Automatically assign an ID to this site if it is missing + if len(site.ID) == 0 { + site.GenerateID() + } + // Infer missing for services for _, service := range site.Services { service.InferMissingData() } } -// GetID generates a unique ID for the site; the following fields are used for this: +// GenerateID generates a unique ID for the site; the following fields are used for this: // Name, Domain -func (site *Site) GetID() string { +func (site *Site) GenerateID() { host := site.Domain if site.Homepage != "" { if hostURL, err := url.Parse(site.Homepage); err == nil { @@ -138,7 +144,7 @@ func (site *Site) GetID() string { } } - return fmt.Sprintf("%s::[%s]", host, site.Name) + site.ID = fmt.Sprintf("%s::[%s]", host, site.Name) } // GetSiteTypeName returns the readable name of the given site type. From 177b3bf0971dc5e9371397c99e8320e84744e287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 Jan 2021 15:55:12 +0100 Subject: [PATCH 15/59] Update documentation and example --- .../config/http/services/mentix/_index.md | 3 ++ .../http/services/mentix/adminapi/_index.md | 46 +++++++++++++++++++ .../http/services/mentix/cs3api/_index.md | 10 +++- .../http/services/mentix/siteloc/_index.md | 10 +++- .../http/services/mentix/webapi/_index.md | 20 +++++++- examples/mentix/mentix.toml | 6 +++ 6 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 docs/content/en/docs/config/http/services/mentix/adminapi/_index.md diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 0f696886a2..8e12bc5be7 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -45,6 +45,9 @@ __Supported importers:__ - **webapi** Mentix can import mesh data via an HTTP endpoint using the `webapi` importer. Data can be sent to the configured relative endpoint (see [here](webapi)). +- **adminapi** + Some aspects of Mentix can be administered through an HTTP endpoint using the `adminapi` importer. Queries can be sent to the configured relative endpoint (see [here](adminapi)). + ## Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. diff --git a/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md b/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md new file mode 100644 index 0000000000..55f29684eb --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md @@ -0,0 +1,46 @@ +--- +title: "adminapi" +linkTitle: "adminapi" +weight: 10 +description: > + Configuration for the AdminAPI of the Mentix service +--- + +{{% pageinfo %}} +The AdminAPI of Mentix is a special importer that can be used to administer certain aspects of Mentix. +{{% /pageinfo %}} + +The AdminAPI importer receives instructions/queries through a `POST` request. + +The importer supports one action that must be passed in the URL: +``` +https://sciencemesh.example.com/mentix/admin/?action= +``` +Currently, the following actions are supported: +- `authorize`: Authorizes or unauthorizes a site + +For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. + +{{% dir name="endpoint" type="string" default="/admin" %}} +The endpoint where the mesh data can be sent to. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +endpoint = "/data" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="" %}} +A list of all enabled connectors for the importer. Must always be provided. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +enabled_connectors = ["localfile"] +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md index d2707bae1c..167de32c17 100644 --- a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md @@ -10,7 +10,7 @@ description: > The CS3API exporter exposes Mentix data in a format that is compliant with the CS3API `ProviderInfo` structure via an HTTP endpoint. {{% /pageinfo %}} -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/cs3" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.cs3api] @@ -18,6 +18,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.cs3api] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md index d33477dbc2..013d8d3f07 100644 --- a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -10,7 +10,7 @@ description: > The Site Locations exporter exposes location information of all sites to be consumed by Grafana via an HTTP endpoint. {{% /pageinfo %}} -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/siteloc" %}} The endpoint where the locations data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.siteloc] @@ -18,6 +18,14 @@ endpoint = "/loc" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.siteloc] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index 9cf4eaea6c..03e4c028bc 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -24,7 +24,7 @@ Currently, the following actions are supported: For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/sites" %}} The endpoint where the mesh data can be sent to. {{< highlight toml >}} [http.services.mentix.importers.webapi] @@ -32,6 +32,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.importers.webapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="" %}} A list of all enabled connectors for the importer. Must always be provided. {{< highlight toml >}} @@ -44,7 +52,7 @@ enabled_connectors = ["localfile"] The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/sites" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.webapi] @@ -52,6 +60,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.webapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index e82c6f8ee1..9baf22bbfc 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -30,6 +30,12 @@ enabled_connectors = ["gocdb"] # For importers, this is obligatory; the connectors will be used as the target for data updates enabled_connectors = ["localfile"] +# Enable the AdminAPI importer +[http.services.mentix.importers.adminapi] +enabled_connectors = ["localfile"] +# Should never allow access w/o prior authorization +is_protected = true + # Configure the Prometheus Service Discovery: [http.services.mentix.exporters.promsd] # The following files must be made available to Prometheus. From 5839218ab4d9491d1ac11af3dc2778d9edadc995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 Jan 2021 16:20:56 +0100 Subject: [PATCH 16/59] Make GenerateId private --- pkg/mentix/meshdata/site.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index ae8cbeb7ee..ce41174ad5 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -125,7 +125,7 @@ func (site *Site) InferMissingData() { // Automatically assign an ID to this site if it is missing if len(site.ID) == 0 { - site.GenerateID() + site.generateID() } // Infer missing for services @@ -134,9 +134,8 @@ func (site *Site) InferMissingData() { } } -// GenerateID generates a unique ID for the site; the following fields are used for this: // Name, Domain -func (site *Site) GenerateID() { +func (site *Site) generateID() { host := site.Domain if site.Homepage != "" { if hostURL, err := url.Parse(site.Homepage); err == nil { From d6a74a2c1c5f323846de02d05628f5932b72a0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 Jan 2021 21:36:38 +0100 Subject: [PATCH 17/59] Fix crash when (un)registering a site --- pkg/mentix/connectors/gocdb.go | 2 +- pkg/mentix/connectors/localfile.go | 6 +++--- pkg/mentix/exchangers/importers/webapi/query.go | 2 +- pkg/mentix/meshdata/properties.go | 9 +++++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index c7d7e80819..75742f27d3 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -130,7 +130,7 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { // Sites coming from the GOCDB are always authorized by default if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 { - meshdata.SetPropertyValue(properties, meshdata.PropertyAuthorized, "true") + meshdata.SetPropertyValue(&properties, meshdata.PropertyAuthorized, "true") } // See if an organization has been defined using properties; otherwise, use the official name diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index e82bf28c07..cbe3da9605 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -131,7 +131,7 @@ func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, upda // Restore the authorization status for all sites for siteID, status := range siteAuthorizationStatus { if site := meshData.FindSite(siteID); site != nil { - meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, status) + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, status) } } return nil @@ -148,9 +148,9 @@ func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, // The site ID is stored in the updated site's name if site := meshData.FindSite(placeholderSite.Name); site != nil { if authorize { - meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "true") + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "true") } else { - meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") } } else { return fmt.Errorf("no site with id '%v' found", placeholderSite.Name) diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index 6a2e89e7f7..868d0c8cc3 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -38,7 +38,7 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { site.ID = "" // Set sites imported through the WebAPI to 'unauthorized' by default - meshdata.SetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} if err := meshData.Verify(); err != nil { diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 1739888300..0374ec2e2f 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -47,6 +47,11 @@ func GetPropertyValue(props map[string]string, id string, defValue string) strin } // SetPropertyValue sets a property value. -func SetPropertyValue(props map[string]string, id string, value string) { - props[strings.ToUpper(id)] = value +func SetPropertyValue(props *map[string]string, id string, value string) { + // If the provided properties map is nil, create an empty one + if *props == nil { + *props = make(map[string]string) + } + + (*props)[strings.ToUpper(id)] = value } From dcc9a946a7d672579351a0517708ce6854966677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 19 Jan 2021 13:26:02 +0100 Subject: [PATCH 18/59] Use site ID field for authorization --- pkg/mentix/connectors/localfile.go | 3 +-- pkg/mentix/exchangers/importers/adminapi/query.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index cbe3da9605..b968c2fe17 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -145,8 +145,7 @@ func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, up func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData, authorize bool) error { for _, placeholderSite := range updatedData.Sites { - // The site ID is stored in the updated site's name - if site := meshData.FindSite(placeholderSite.Name); site != nil { + if site := meshData.FindSite(placeholderSite.ID); site != nil { if authorize { meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "true") } else { diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go index 36c2265849..cb7a65a198 100755 --- a/pkg/mentix/exchangers/importers/adminapi/query.go +++ b/pkg/mentix/exchangers/importers/adminapi/query.go @@ -38,7 +38,7 @@ func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { if value, ok := jsonData["id"]; ok { if id, ok := value.(string); ok { site := &meshdata.Site{} - site.Name = id // Store the provided site ID in the site's name + site.ID = id // We only need to store the ID of the site meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} return meshData, nil From ea1269d5abaee7e27b1e21aa21f0439e0c074bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 5 Feb 2021 13:07:54 +0100 Subject: [PATCH 19/59] Fix Mentix data cloning bug --- changelog/unreleased/mentix-clone-fix.md | 5 +++++ pkg/mentix/connectors/localfile.go | 1 + pkg/mentix/meshdata/meshdata.go | 23 +++++++++++------------ 3 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 changelog/unreleased/mentix-clone-fix.md diff --git a/changelog/unreleased/mentix-clone-fix.md b/changelog/unreleased/mentix-clone-fix.md new file mode 100644 index 0000000000..635f49064d --- /dev/null +++ b/changelog/unreleased/mentix-clone-fix.md @@ -0,0 +1,5 @@ +Fix: Cloning of internal mesh data lost some values + +This update fixes a bug in Mentix that caused some (non-critical) values to be lost during data cloning that happens internally. + +https://github.com/cs3org/reva/pull/1457 diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index b968c2fe17..6c154adc2c 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -79,6 +79,7 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err connector.setSiteTypes(meshData) meshData.InferMissingData() + return meshData, nil } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 9ab0ecdf4a..05289b72d1 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -19,8 +19,11 @@ package meshdata import ( + "bytes" + "encoding/gob" "encoding/json" "fmt" + "reflect" "strings" ) @@ -196,10 +199,13 @@ func (meshData *MeshData) FromJSON(data string) error { func (meshData *MeshData) Clone() *MeshData { clone := &MeshData{} - // To avoid any "deep copy" packages, use JSON en- and decoding instead - data, err := meshData.ToJSON() - if err == nil { - if err := clone.FromJSON(data); err != nil { + // To avoid any "deep copy" packages, use gob en- and decoding instead + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + + if err := enc.Encode(meshData); err == nil { + if err := dec.Decode(clone); err != nil { // In case of an error, clear the data clone.Clear() } @@ -210,14 +216,7 @@ func (meshData *MeshData) Clone() *MeshData { // Compare checks whether the stored data equals the data of another MeshData object. func (meshData *MeshData) Compare(other *MeshData) bool { - if other == nil { - return false - } - - // To avoid cumbersome comparisons, just compare the JSON-encoded data - json1, _ := meshData.ToJSON() - json2, _ := other.ToJSON() - return json1 == json2 + return reflect.DeepEqual(meshData, other) } // New returns a new (empty) MeshData object. From 7ab6db27240b65a986359328287caf7004a46f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 5 Feb 2021 13:32:27 +0100 Subject: [PATCH 20/59] Fix changelog --- changelog/unreleased/mentix-clone-fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/mentix-clone-fix.md b/changelog/unreleased/mentix-clone-fix.md index 635f49064d..d5ab1f3232 100644 --- a/changelog/unreleased/mentix-clone-fix.md +++ b/changelog/unreleased/mentix-clone-fix.md @@ -1,4 +1,4 @@ -Fix: Cloning of internal mesh data lost some values +Bugfix: Cloning of internal mesh data lost some values This update fixes a bug in Mentix that caused some (non-critical) values to be lost during data cloning that happens internally. From 26495e07b185e08fb6a0884cfecf5498d0e3ce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 16 Feb 2021 16:33:05 +0100 Subject: [PATCH 21/59] Begin work on new accounts service --- internal/http/services/accounts/accounts.go | 177 ++++++++++++++++++ .../http/services/accounts/config/config.go | 32 ++++ .../services/accounts/config/endpoints.go | 26 +++ internal/http/services/accounts/data/data.go | 69 +++++++ .../services/accounts/data/filestorage.go | 113 +++++++++++ .../http/services/accounts/data/storage.go | 34 ++++ internal/http/services/accounts/manager.go | 126 +++++++++++++ internal/http/services/loader/loader.go | 1 + pkg/apikey/apikey.go | 22 +++ pkg/utils/utils.go | 9 + 10 files changed, 609 insertions(+) create mode 100644 internal/http/services/accounts/accounts.go create mode 100644 internal/http/services/accounts/config/config.go create mode 100644 internal/http/services/accounts/config/endpoints.go create mode 100644 internal/http/services/accounts/data/data.go create mode 100644 internal/http/services/accounts/data/filestorage.go create mode 100644 internal/http/services/accounts/data/storage.go create mode 100644 internal/http/services/accounts/manager.go create mode 100644 pkg/apikey/apikey.go diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go new file mode 100644 index 0000000000..ddb1502b6b --- /dev/null +++ b/internal/http/services/accounts/accounts.go @@ -0,0 +1,177 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package accounts + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/internal/http/services/accounts/config" + "github.com/cs3org/reva/pkg/rhttp/global" +) + +func init() { + global.Register(serviceName, New) +} + +type svc struct { + conf *config.Configuration + log *zerolog.Logger + + manager *Manager +} + +const ( + serviceName = "accounts" +) + +// Close is called when this service is being stopped. +func (s *svc) Close() error { + return nil +} + +// Prefix returns the main endpoint of this service. +func (s *svc) Prefix() string { + return s.conf.Prefix +} + +// Unprotected returns all endpoints that can be queried without prior authorization. +func (s *svc) Unprotected() []string { + // TODO: For testing only + return []string{"/"} + // This service currently only has one public endpoint (called "register") used for account registration + return []string{config.EndpointRegister} +} + +// Handler serves all HTTP requests. +func (s *svc) Handler() http.Handler { + // Allow definition of endpoints in a flexible and easy way + type Endpoint struct { + Path string + Method string + Handler func(url.Values, []byte) (interface{}, error) + } + + endpoints := []Endpoint{ + {config.EndpointList, http.MethodGet, s.handleList}, + {config.EndpointRegister, http.MethodPost, s.handleRegister}, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + // Every request to the accounts service results in a standardized JSON response + type Response struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Data interface{} `json:"data,omitempty"` + } + + // The default response is an unknown endpoint (for the specified method) + resp := Response{ + Success: false, + Error: fmt.Sprintf("unknown endpoint %v for method %v", r.URL.Path, r.Method), + Data: nil, + } + + // Check each endpoint if it can handle the request + for _, endpoint := range endpoints { + if r.URL.Path == endpoint.Path && r.Method == endpoint.Method { + body, _ := ioutil.ReadAll(r.Body) + + if data, err := endpoint.Handler(r.URL.Query(), body); err == nil { + resp.Success = true + resp.Error = "" + resp.Data = data + } else { + resp.Success = false + resp.Error = fmt.Sprintf("%v", err) + resp.Data = nil + } + + break + } + } + + // Any failure during query handling results in a bad request + if !resp.Success { + w.WriteHeader(http.StatusBadRequest) + } + + jsonData, _ := json.MarshalIndent(&resp, "", "\t") + _, _ = w.Write(jsonData) + }) +} + +func (s *svc) handleList(values url.Values, data []byte) (interface{}, error) { + return s.manager.ClonedAccounts(), nil +} + +func (s *svc) handleRegister(values url.Values, data []byte) (interface{}, error) { + return map[string]string{"id": "okiii"}, nil +} + +func parseConfig(m map[string]interface{}) (*config.Configuration, error) { + conf := &config.Configuration{} + if err := mapstructure.Decode(m, &conf); err != nil { + return nil, errors.Wrap(err, "error decoding configuration") + } + applyDefaultConfig(conf) + return conf, nil +} + +func applyDefaultConfig(conf *config.Configuration) { + if conf.Prefix == "" { + conf.Prefix = serviceName + } + + if conf.Storage.Driver == "" { + conf.Storage.Driver = "file" + } +} + +// New returns a new Accounts service. +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { + // Prepare the configuration + conf, err := parseConfig(m) + if err != nil { + return nil, err + } + + // Create the accounts manager instance + mngr, err := newManager(conf, log) + if err != nil { + return nil, errors.Wrap(err, "error creating the accounts service") + } + + // Create the service + s := &svc{ + conf: conf, + log: log, + manager: mngr, + } + return s, nil +} diff --git a/internal/http/services/accounts/config/config.go b/internal/http/services/accounts/config/config.go new file mode 100644 index 0000000000..7861701968 --- /dev/null +++ b/internal/http/services/accounts/config/config.go @@ -0,0 +1,32 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +// Configuration holds the general service configuration. +type Configuration struct { + Prefix string `mapstructure:"prefix"` + + Storage struct { + Driver string `mapstructure:"driver"` + + File struct { + File string `mapstructure:"file"` + } `mapstructure:"file"` + } `mapstructure:"storage"` +} diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go new file mode 100644 index 0000000000..c6cb64bb26 --- /dev/null +++ b/internal/http/services/accounts/config/endpoints.go @@ -0,0 +1,26 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +const ( + // EndpointList is the endpoint path for listing all stored accounts. + EndpointList = "/list" + // EndpointRegister is the endpoint path for account registration. + EndpointRegister = "/register" +) diff --git a/internal/http/services/accounts/data/data.go b/internal/http/services/accounts/data/data.go new file mode 100644 index 0000000000..1ec6a2403d --- /dev/null +++ b/internal/http/services/accounts/data/data.go @@ -0,0 +1,69 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this filePath except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package data + +import ( + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/apikey" + "github.com/cs3org/reva/pkg/utils" +) + +// Account represents a single user account. +type Account struct { + Email string `json:"email"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` + + Data AccountData `json:"data"` +} + +// AccountData holds additional data for a user account. +type AccountData struct { + APIKey apikey.APIKey `json:"apiKey"` + Authorized bool `json:"authorized"` +} + +// Accounts holds an array of user accounts. +type Accounts = []*Account + +// NewAccount creates a new user account. +func NewAccount(email string, firstName, lastName string) (*Account, error) { + if email == "" { + return nil, errors.Errorf("no email address provided") + } else if !utils.IsEmailValid(email) { + return nil, errors.Errorf("invalid email address: %v", email) + } + + if firstName == "" || lastName == "" { + return nil, errors.Errorf("no or incomplete name provided") + } + + acc := &Account{ + Email: email, + FirstName: firstName, + LastName: lastName, + Data: AccountData{ + APIKey: "", + Authorized: false, + }, + } + + return acc, nil +} diff --git a/internal/http/services/accounts/data/filestorage.go b/internal/http/services/accounts/data/filestorage.go new file mode 100644 index 0000000000..b240af8de3 --- /dev/null +++ b/internal/http/services/accounts/data/filestorage.go @@ -0,0 +1,113 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this filePath except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package data + +import ( + "encoding/json" + "io/ioutil" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/internal/http/services/accounts/config" +) + +// FileStorage implements a filePath-based storage. +type FileStorage struct { + Storage + + conf *config.Configuration + log *zerolog.Logger + + filePath string +} + +func (storage *FileStorage) initialize(conf *config.Configuration, log *zerolog.Logger) error { + if conf == nil { + return errors.Errorf("no configuration provided") + } + storage.conf = conf + + if log == nil { + return errors.Errorf("no logger provided") + } + storage.log = log + + if conf.Storage.File.File == "" { + return errors.Errorf("no file set in the configuration") + } + storage.filePath = conf.Storage.File.File + + return nil +} + +// ReadAll reads all stored accounts into the given data object. +func (storage *FileStorage) ReadAll() (*Accounts, error) { + accounts := &Accounts{} + + // Read the data from the configured file + jsonData, err := ioutil.ReadFile(storage.filePath) + if err != nil { + return nil, errors.Wrapf(err, "unable to read file %v", storage.filePath) + } + + if err := json.Unmarshal(jsonData, accounts); err != nil { + return nil, errors.Wrapf(err, "invalid file %v", storage.filePath) + } + + return accounts, nil +} + +// WriteAll writes all stored accounts from the given data object. +func (storage *FileStorage) WriteAll(accounts *Accounts) error { + // Write the data to the configured file + jsonData, _ := json.MarshalIndent(accounts, "", "\t") + if err := ioutil.WriteFile(storage.filePath, jsonData, 0755); err != nil { + return errors.Wrapf(err, "unable to write file %v", storage.filePath) + } + + return nil +} + +// AccountAdded is called when an account has been added. +func (storage *FileStorage) AccountAdded(account *Account) error { + // Simply skip this action; all data is saved solely in WriteAll + return nil +} + +// AccountAdded is called when an account has been updated. +func (storage *FileStorage) AccountUpdated(account *Account) error { + // Simply skip this action; all data is saved solely in WriteAll + return nil +} + +// AccountAdded is called when an account has been removed. +func (storage *FileStorage) AccountRemoved(account *Account) error { + // Simply skip this action; all data is saved solely in WriteAll + return nil +} + +// NewFileStorage creates a new filePath storage. +func NewFileStorage(conf *config.Configuration, log *zerolog.Logger) (*FileStorage, error) { + storage := &FileStorage{} + if err := storage.initialize(conf, log); err != nil { + return nil, errors.Wrapf(err, "unable to initialize the filePath storage") + } + return storage, nil +} diff --git a/internal/http/services/accounts/data/storage.go b/internal/http/services/accounts/data/storage.go new file mode 100644 index 0000000000..ffbd5e64c6 --- /dev/null +++ b/internal/http/services/accounts/data/storage.go @@ -0,0 +1,34 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this filePath except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package data + +// Storage defines the interface for accounts storages. +type Storage interface { + // ReadAll reads all stored accounts into the given data object. + ReadAll() (*Accounts, error) + // WriteAll writes all stored accounts from the given data object. + WriteAll(accounts *Accounts) error + + // AccountAdded is called when an account has been added. + AccountAdded(account *Account) error + // AccountAdded is called when an account has been updated. + AccountUpdated(account *Account) error + // AccountAdded is called when an account has been removed. + AccountRemoved(account *Account) error +} diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go new file mode 100644 index 0000000000..6f83c84ce1 --- /dev/null +++ b/internal/http/services/accounts/manager.go @@ -0,0 +1,126 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package accounts + +import ( + "bytes" + "encoding/gob" + "sync" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/internal/http/services/accounts/config" + "github.com/cs3org/reva/internal/http/services/accounts/data" +) + +// Manager is responsible for all user account related tasks. +type Manager struct { + conf *config.Configuration + log *zerolog.Logger + + accounts data.Accounts + storage data.Storage + + mutex sync.RWMutex +} + +func (mngr *Manager) initialize(conf *config.Configuration, log *zerolog.Logger) error { + if conf == nil { + return errors.Errorf("no configuration provided") + } + mngr.conf = conf + + if log == nil { + return errors.Errorf("no logger provided") + } + mngr.log = log + + mngr.accounts = make(data.Accounts, 0, 32) // Reserve some space for accounts + + // Create the user accounts storage and read all stored data + if storage, err := mngr.createStorage(conf.Storage.Driver); err == nil { + mngr.storage = storage + mngr.readAllAccounts() + } else { + return errors.Wrap(err, "unable to create accounts storage") + } + + return nil +} + +func (mngr *Manager) createStorage(driver string) (data.Storage, error) { + if driver == "file" { + return data.NewFileStorage(mngr.conf, mngr.log) + } + + return nil, errors.Errorf("unknown storage driver %v", driver) +} + +func (mngr *Manager) readAllAccounts() { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + if accounts, err := mngr.storage.ReadAll(); err == nil { + mngr.accounts = *accounts + } else { + // Just warn when not being able to read accounts + mngr.log.Warn().Err(err).Msg("error while reading accounts") + } +} + +func (mngr *Manager) writeAllAccounts() { + mngr.mutex.RLock() + defer mngr.mutex.RUnlock() + + if err := mngr.storage.WriteAll(&mngr.accounts); err != nil { + // Just warn when not being able to write accounts + mngr.log.Warn().Err(err).Msg("error while writing accounts") + } +} + +// ClonedAccounts retrieves all accounts currently stored by cloning the data, thus avoiding race conflicts and making outside modifications impossible. +func (mngr *Manager) ClonedAccounts() data.Accounts { + mngr.mutex.RLock() + defer mngr.mutex.RUnlock() + + clone := make(data.Accounts, 0) + + // To avoid any "deep copy" packages, use gob en- and decoding instead + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + + if err := enc.Encode(mngr.accounts); err == nil { + if err := dec.Decode(&clone); err != nil { + // In case of an error, create an empty data set + clone = make(data.Accounts, 0) + } + } + + return clone +} + +func newManager(conf *config.Configuration, log *zerolog.Logger) (*Manager, error) { + mngr := &Manager{} + if err := mngr.initialize(conf, log); err != nil { + return nil, errors.Wrapf(err, "unable to initialize the accounts manager") + } + return mngr, nil +} diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 0013049632..4a5c12ab98 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -20,6 +20,7 @@ package loader import ( // Load core HTTP services + _ "github.com/cs3org/reva/internal/http/services/accounts" _ "github.com/cs3org/reva/internal/http/services/datagateway" _ "github.com/cs3org/reva/internal/http/services/dataprovider" _ "github.com/cs3org/reva/internal/http/services/helloworld" diff --git a/pkg/apikey/apikey.go b/pkg/apikey/apikey.go new file mode 100644 index 0000000000..2727ab15c8 --- /dev/null +++ b/pkg/apikey/apikey.go @@ -0,0 +1,22 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package apikey + +// APIKey is the type used to store API keys. +type APIKey = string diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index ad2a9cbcd8..c7c495e883 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -36,6 +36,7 @@ import ( var ( matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") + matchEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") ) // Skip evaluates whether a source endpoint contains any of the prefixes. @@ -146,3 +147,11 @@ func GranteeEqual(u, v *provider.Grantee) bool { vu, vg := ExtractGranteeID(v) return u.Type == v.Type && (UserEqual(uu, vu) || GroupEqual(ug, vg)) } + +// IsEmailValid checks whether the provided email has a valid format. +func IsEmailValid(e string) bool { + if len(e) < 3 && len(e) > 254 { + return false + } + return matchEmail.MatchString(e) +} From 0516ceb75616a5c9221bcc32c1e8d8fc16e1961d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 16 Feb 2021 17:37:16 +0100 Subject: [PATCH 22/59] Implement account creation, update, removal endpoints --- internal/http/services/accounts/accounts.go | 53 ++++++++-- .../services/accounts/config/endpoints.go | 8 +- internal/http/services/accounts/data/data.go | 41 ++++++-- .../services/accounts/data/filestorage.go | 9 +- .../http/services/accounts/data/storage.go | 6 +- internal/http/services/accounts/manager.go | 98 +++++++++++++++++-- 6 files changed, 182 insertions(+), 33 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index ddb1502b6b..791bf0b95d 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -30,6 +30,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/internal/http/services/accounts/config" + "github.com/cs3org/reva/internal/http/services/accounts/data" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -63,7 +64,7 @@ func (s *svc) Unprotected() []string { // TODO: For testing only return []string{"/"} // This service currently only has one public endpoint (called "register") used for account registration - return []string{config.EndpointRegister} + return []string{config.EndpointCreate} } // Handler serves all HTTP requests. @@ -77,7 +78,9 @@ func (s *svc) Handler() http.Handler { endpoints := []Endpoint{ {config.EndpointList, http.MethodGet, s.handleList}, - {config.EndpointRegister, http.MethodPost, s.handleRegister}, + {config.EndpointCreate, http.MethodPost, s.handleCreate}, + {config.EndpointUpdate, http.MethodPost, s.handleUpdate}, + {config.EndpointRemove, http.MethodPost, s.handleRemove}, } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -126,12 +129,50 @@ func (s *svc) Handler() http.Handler { }) } -func (s *svc) handleList(values url.Values, data []byte) (interface{}, error) { - return s.manager.ClonedAccounts(), nil +func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { + return s.manager.CloneAccounts(), nil } -func (s *svc) handleRegister(values url.Values, data []byte) (interface{}, error) { - return map[string]string{"id": "okiii"}, nil +func (s *svc) handleCreate(values url.Values, body []byte) (interface{}, error) { + account := &data.Account{} + if err := json.Unmarshal(body, account); err != nil { + return nil, errors.Wrap(err, "invalid account data") + } + + // Create a new account through the account manager + if err := s.manager.CreateAccount(account); err != nil { + return nil, errors.Wrap(err, "unable to create account") + } + + return nil, nil +} + +func (s *svc) handleUpdate(values url.Values, body []byte) (interface{}, error) { + account := &data.Account{} + if err := json.Unmarshal(body, account); err != nil { + return nil, errors.Wrap(err, "invalid account data") + } + + // Update the account through the account manager; only the basic data of an account can be updated through this endpoint + if err := s.manager.UpdateAccount(account, false); err != nil { + return nil, errors.Wrap(err, "unable to update account") + } + + return nil, nil +} + +func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) { + account := &data.Account{} + if err := json.Unmarshal(body, account); err != nil { + return nil, errors.Wrap(err, "invalid account data") + } + + // Remove the account through the account manager + if err := s.manager.RemoveAccount(account); err != nil { + return nil, errors.Wrap(err, "unable to remove account") + } + + return nil, nil } func parseConfig(m map[string]interface{}) (*config.Configuration, error) { diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index c6cb64bb26..6b0f2ef07b 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -21,6 +21,10 @@ package config const ( // EndpointList is the endpoint path for listing all stored accounts. EndpointList = "/list" - // EndpointRegister is the endpoint path for account registration. - EndpointRegister = "/register" + // EndpointCreate is the endpoint path for account creation. + EndpointCreate = "/create" + // EndpointUpdate is the endpoint path for account updates. + EndpointUpdate = "/update" + // EndpointRemove is the endpoint path for account removal. + EndpointRemove = "/remove" ) diff --git a/internal/http/services/accounts/data/data.go b/internal/http/services/accounts/data/data.go index 1ec6a2403d..22c7aaf8c3 100644 --- a/internal/http/services/accounts/data/data.go +++ b/internal/http/services/accounts/data/data.go @@ -43,18 +43,39 @@ type AccountData struct { // Accounts holds an array of user accounts. type Accounts = []*Account -// NewAccount creates a new user account. -func NewAccount(email string, firstName, lastName string) (*Account, error) { - if email == "" { - return nil, errors.Errorf("no email address provided") - } else if !utils.IsEmailValid(email) { - return nil, errors.Errorf("invalid email address: %v", email) +// Copy copies the data of the given account to this account; if copyData is true, the account data is copied as well. +func (acc *Account) Copy(other *Account, copyData bool) error { + if err := other.verify(); err != nil { + return errors.Wrap(err, "unable to update account data") + } + + // Manually update fields + acc.FirstName = other.FirstName + acc.LastName = other.LastName + + if copyData { + acc.Data = other.Data } - if firstName == "" || lastName == "" { - return nil, errors.Errorf("no or incomplete name provided") + return nil +} + +func (acc *Account) verify() error { + if acc.Email == "" { + return errors.Errorf("no email address provided") + } else if !utils.IsEmailValid(acc.Email) { + return errors.Errorf("invalid email address: %v", acc.Email) } + if acc.FirstName == "" || acc.LastName == "" { + return errors.Errorf("no or incomplete name provided") + } + + return nil +} + +// NewAccount creates a new user account. +func NewAccount(email string, firstName, lastName string) (*Account, error) { acc := &Account{ Email: email, FirstName: firstName, @@ -65,5 +86,9 @@ func NewAccount(email string, firstName, lastName string) (*Account, error) { }, } + if err := acc.verify(); err != nil { + return nil, err + } + return acc, nil } diff --git a/internal/http/services/accounts/data/filestorage.go b/internal/http/services/accounts/data/filestorage.go index b240af8de3..681e133758 100644 --- a/internal/http/services/accounts/data/filestorage.go +++ b/internal/http/services/accounts/data/filestorage.go @@ -86,21 +86,18 @@ func (storage *FileStorage) WriteAll(accounts *Accounts) error { } // AccountAdded is called when an account has been added. -func (storage *FileStorage) AccountAdded(account *Account) error { +func (storage *FileStorage) AccountAdded(account *Account) { // Simply skip this action; all data is saved solely in WriteAll - return nil } // AccountAdded is called when an account has been updated. -func (storage *FileStorage) AccountUpdated(account *Account) error { +func (storage *FileStorage) AccountUpdated(account *Account) { // Simply skip this action; all data is saved solely in WriteAll - return nil } // AccountAdded is called when an account has been removed. -func (storage *FileStorage) AccountRemoved(account *Account) error { +func (storage *FileStorage) AccountRemoved(account *Account) { // Simply skip this action; all data is saved solely in WriteAll - return nil } // NewFileStorage creates a new filePath storage. diff --git a/internal/http/services/accounts/data/storage.go b/internal/http/services/accounts/data/storage.go index ffbd5e64c6..58269c72ba 100644 --- a/internal/http/services/accounts/data/storage.go +++ b/internal/http/services/accounts/data/storage.go @@ -26,9 +26,9 @@ type Storage interface { WriteAll(accounts *Accounts) error // AccountAdded is called when an account has been added. - AccountAdded(account *Account) error + AccountAdded(account *Account) // AccountAdded is called when an account has been updated. - AccountUpdated(account *Account) error + AccountUpdated(account *Account) // AccountAdded is called when an account has been removed. - AccountRemoved(account *Account) error + AccountRemoved(account *Account) } diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 6f83c84ce1..0508a8402b 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -21,6 +21,7 @@ package accounts import ( "bytes" "encoding/gob" + "strings" "sync" "github.com/pkg/errors" @@ -28,6 +29,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/pkg/apikey" ) // Manager is responsible for all user account related tasks. @@ -74,9 +76,6 @@ func (mngr *Manager) createStorage(driver string) (data.Storage, error) { } func (mngr *Manager) readAllAccounts() { - mngr.mutex.Lock() - defer mngr.mutex.Unlock() - if accounts, err := mngr.storage.ReadAll(); err == nil { mngr.accounts = *accounts } else { @@ -86,17 +85,100 @@ func (mngr *Manager) readAllAccounts() { } func (mngr *Manager) writeAllAccounts() { - mngr.mutex.RLock() - defer mngr.mutex.RUnlock() - if err := mngr.storage.WriteAll(&mngr.accounts); err != nil { // Just warn when not being able to write accounts mngr.log.Warn().Err(err).Msg("error while writing accounts") } } -// ClonedAccounts retrieves all accounts currently stored by cloning the data, thus avoiding race conflicts and making outside modifications impossible. -func (mngr *Manager) ClonedAccounts() data.Accounts { +func (mngr *Manager) findAccountByEmail(email string) *data.Account { + if email == "" { + return nil + } + + // Perform a case-insensitive search of the given email address + for _, account := range mngr.accounts { + if strings.EqualFold(account.Email, email) { + return account + } + } + return nil +} + +func (mngr *Manager) findAccountByAPIKey(key apikey.APIKey) *data.Account { + if key == "" { + return nil + } + + // Perform a case-sensitive search of the given API key + for _, account := range mngr.accounts { + if account.Data.APIKey == key { + return account + } + } + return nil +} + +// CreateAccount creates a new account; if an account with the same email address already exists, an error is returned. +func (mngr *Manager) CreateAccount(accountData *data.Account) error { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + // Accounts must be unique (identified by their email address) + if mngr.findAccountByEmail(accountData.Email) != nil { + return errors.Errorf("an account with the specified email address already exists") + } + + if account, err := data.NewAccount(accountData.Email, accountData.FirstName, accountData.LastName); err == nil { + mngr.accounts = append(mngr.accounts, account) + mngr.storage.AccountAdded(account) + mngr.writeAllAccounts() + } else { + return errors.Wrap(err, "error while creating account") + } + + return nil +} + +// UpdateAccount updates the account identified by the account email; if no such account exists, an error is returned. +func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) error { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + account := mngr.findAccountByEmail(accountData.Email) + if account == nil { + return errors.Errorf("no account with the specified email exists") + } + + if err := account.Copy(accountData, copyData); err == nil { + mngr.storage.AccountUpdated(account) + mngr.writeAllAccounts() + } else { + return errors.Wrap(err, "error while updating account") + } + + return nil +} + +// RemoveAccount removes the account identified by the account email; if no such account exists, an error is returned. +func (mngr *Manager) RemoveAccount(accountData *data.Account) error { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + for i, account := range mngr.accounts { + if strings.EqualFold(account.Email, accountData.Email) { + mngr.accounts = append(mngr.accounts[:i], mngr.accounts[i+1:]...) + mngr.storage.AccountRemoved(account) + mngr.writeAllAccounts() + return nil + } + } + + return errors.Errorf("no account with the specified email exists") +} + +// CloneAccounts retrieves all accounts currently stored by cloning the data, thus avoiding race conflicts and making outside modifications impossible. +func (mngr *Manager) CloneAccounts() data.Accounts { mngr.mutex.RLock() defer mngr.mutex.RUnlock() From 7be2952b8150b743911626eb65ab2b90d7f9a439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 17 Feb 2021 15:00:13 +0100 Subject: [PATCH 23/59] Add web interface panel --- internal/http/services/accounts/accounts.go | 153 ++++++++++++------ .../services/accounts/config/endpoints.go | 7 + internal/http/services/accounts/data/data.go | 15 +- internal/http/services/accounts/manager.go | 38 +++++ .../http/services/accounts/panel/panel.go | 81 ++++++++++ .../http/services/accounts/panel/template.go | 106 ++++++++++++ 6 files changed, 349 insertions(+), 51 deletions(-) create mode 100644 internal/http/services/accounts/panel/panel.go create mode 100644 internal/http/services/accounts/panel/template.go diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 791bf0b95d..03271c273e 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -69,6 +70,27 @@ func (s *svc) Unprotected() []string { // Handler serves all HTTP requests. func (s *svc) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + switch r.URL.Path { + case config.EndpointPanel: + s.handlePanelEndpoint(w, r) + + default: + s.handleRequestEndpoints(w, r) + } + }) +} + +func (s *svc) handlePanelEndpoint(w http.ResponseWriter, r *http.Request) { + if err := s.manager.ShowPanel(w); err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(fmt.Sprintf("Unable to show the web interface panel: %v", err))) + } +} + +func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { // Allow definition of endpoints in a flexible and easy way type Endpoint struct { Path string @@ -76,57 +98,54 @@ func (s *svc) Handler() http.Handler { Handler func(url.Values, []byte) (interface{}, error) } + // Every request to the accounts service results in a standardized JSON response + type Response struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Data interface{} `json:"data,omitempty"` + } + endpoints := []Endpoint{ {config.EndpointList, http.MethodGet, s.handleList}, {config.EndpointCreate, http.MethodPost, s.handleCreate}, {config.EndpointUpdate, http.MethodPost, s.handleUpdate}, {config.EndpointRemove, http.MethodPost, s.handleRemove}, + {config.EndpointAuthorize, http.MethodPost, s.handleAuthorize}, } - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - // Every request to the accounts service results in a standardized JSON response - type Response struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - Data interface{} `json:"data,omitempty"` - } - - // The default response is an unknown endpoint (for the specified method) - resp := Response{ - Success: false, - Error: fmt.Sprintf("unknown endpoint %v for method %v", r.URL.Path, r.Method), - Data: nil, - } + // The default response is an unknown endpoint (for the specified method) + resp := Response{ + Success: false, + Error: fmt.Sprintf("unknown endpoint %v for method %v", r.URL.Path, r.Method), + Data: nil, + } - // Check each endpoint if it can handle the request - for _, endpoint := range endpoints { - if r.URL.Path == endpoint.Path && r.Method == endpoint.Method { - body, _ := ioutil.ReadAll(r.Body) - - if data, err := endpoint.Handler(r.URL.Query(), body); err == nil { - resp.Success = true - resp.Error = "" - resp.Data = data - } else { - resp.Success = false - resp.Error = fmt.Sprintf("%v", err) - resp.Data = nil - } - - break + // Check each endpoint if it can handle the request + for _, endpoint := range endpoints { + if r.URL.Path == endpoint.Path && r.Method == endpoint.Method { + body, _ := ioutil.ReadAll(r.Body) + + if data, err := endpoint.Handler(r.URL.Query(), body); err == nil { + resp.Success = true + resp.Error = "" + resp.Data = data + } else { + resp.Success = false + resp.Error = fmt.Sprintf("%v", err) + resp.Data = nil } - } - // Any failure during query handling results in a bad request - if !resp.Success { - w.WriteHeader(http.StatusBadRequest) + break } + } - jsonData, _ := json.MarshalIndent(&resp, "", "\t") - _, _ = w.Write(jsonData) - }) + // Any failure during query handling results in a bad request + if !resp.Success { + w.WriteHeader(http.StatusBadRequest) + } + + jsonData, _ := json.MarshalIndent(&resp, "", "\t") + _, _ = w.Write(jsonData) } func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { @@ -134,9 +153,9 @@ func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { } func (s *svc) handleCreate(values url.Values, body []byte) (interface{}, error) { - account := &data.Account{} - if err := json.Unmarshal(body, account); err != nil { - return nil, errors.Wrap(err, "invalid account data") + account, err := s.unmarshalRequestData(body) + if err != nil { + return nil, err } // Create a new account through the account manager @@ -148,9 +167,9 @@ func (s *svc) handleCreate(values url.Values, body []byte) (interface{}, error) } func (s *svc) handleUpdate(values url.Values, body []byte) (interface{}, error) { - account := &data.Account{} - if err := json.Unmarshal(body, account); err != nil { - return nil, errors.Wrap(err, "invalid account data") + account, err := s.unmarshalRequestData(body) + if err != nil { + return nil, err } // Update the account through the account manager; only the basic data of an account can be updated through this endpoint @@ -162,9 +181,9 @@ func (s *svc) handleUpdate(values url.Values, body []byte) (interface{}, error) } func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) { - account := &data.Account{} - if err := json.Unmarshal(body, account); err != nil { - return nil, errors.Wrap(err, "invalid account data") + account, err := s.unmarshalRequestData(body) + if err != nil { + return nil, err } // Remove the account through the account manager @@ -175,6 +194,44 @@ func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) return nil, nil } +func (s *svc) handleAuthorize(values url.Values, body []byte) (interface{}, error) { + account, err := s.unmarshalRequestData(body) + if err != nil { + return nil, err + } + + if val, ok := values["status"]; ok && len(val) > 0 { + var authorize bool + switch strings.ToLower(val[0]) { + case "true": + authorize = true + + case "false": + authorize = false + + default: + return nil, errors.Errorf("unsupported authorization status %v", val[0]) + } + + // Authorize the account through the account manager + if err := s.manager.AuthorizeAccount(account, authorize); err != nil { + return nil, errors.Wrap(err, "unable to remove account") + } + } else { + return nil, errors.Errorf("no authorization status provided") + } + + return nil, nil +} + +func (s *svc) unmarshalRequestData(body []byte) (*data.Account, error) { + account := &data.Account{} + if err := json.Unmarshal(body, account); err != nil { + return nil, errors.Wrap(err, "invalid account data") + } + return account, nil +} + func parseConfig(m map[string]interface{}) (*config.Configuration, error) { conf := &config.Configuration{} if err := mapstructure.Decode(m, &conf); err != nil { diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index 6b0f2ef07b..f6077ca6c7 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -19,12 +19,19 @@ package config const ( + // EndpointPanel is the endpoint of the web interface panel. + EndpointPanel = "/panel" + // EndpointList is the endpoint path for listing all stored accounts. EndpointList = "/list" + // EndpointCreate is the endpoint path for account creation. EndpointCreate = "/create" // EndpointUpdate is the endpoint path for account updates. EndpointUpdate = "/update" // EndpointRemove is the endpoint path for account removal. EndpointRemove = "/remove" + + // EndpointAuthorize is the endpoint path for account authorization + EndpointAuthorize = "/authorize" ) diff --git a/internal/http/services/accounts/data/data.go b/internal/http/services/accounts/data/data.go index 22c7aaf8c3..9f6b5b5639 100644 --- a/internal/http/services/accounts/data/data.go +++ b/internal/http/services/accounts/data/data.go @@ -19,6 +19,8 @@ package data import ( + "time" + "github.com/pkg/errors" "github.com/cs3org/reva/pkg/apikey" @@ -31,6 +33,9 @@ type Account struct { FirstName string `json:"firstName"` LastName string `json:"lastName"` + DateCreated time.Time `json:"dateCreated"` + DateModified time.Time `json:"dateModified"` + Data AccountData `json:"data"` } @@ -76,10 +81,14 @@ func (acc *Account) verify() error { // NewAccount creates a new user account. func NewAccount(email string, firstName, lastName string) (*Account, error) { + t := time.Now() + acc := &Account{ - Email: email, - FirstName: firstName, - LastName: lastName, + Email: email, + FirstName: firstName, + LastName: lastName, + DateCreated: t, + DateModified: t, Data: AccountData{ APIKey: "", Authorized: false, diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 0508a8402b..0dcf4704b2 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -21,14 +21,17 @@ package accounts import ( "bytes" "encoding/gob" + "net/http" "strings" "sync" + "time" "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/internal/http/services/accounts/panel" "github.com/cs3org/reva/pkg/apikey" ) @@ -40,6 +43,8 @@ type Manager struct { accounts data.Accounts storage data.Storage + panel *panel.Panel + mutex sync.RWMutex } @@ -64,6 +69,13 @@ func (mngr *Manager) initialize(conf *config.Configuration, log *zerolog.Logger) return errors.Wrap(err, "unable to create accounts storage") } + // Create the web interface panel + if pnl, err := panel.NewPanel(conf, log); err == nil { + mngr.panel = pnl + } else { + return errors.Wrap(err, "unable to create panel") + } + return nil } @@ -119,6 +131,12 @@ func (mngr *Manager) findAccountByAPIKey(key apikey.APIKey) *data.Account { return nil } +func (mngr *Manager) ShowPanel(w http.ResponseWriter) error { + // The panel only shows the stored accounts and offers actions through links, so let it use cloned data + accounts := mngr.CloneAccounts() + return mngr.panel.Execute(w, &accounts) +} + // CreateAccount creates a new account; if an account with the same email address already exists, an error is returned. func (mngr *Manager) CreateAccount(accountData *data.Account) error { mngr.mutex.Lock() @@ -151,6 +169,8 @@ func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) err } if err := account.Copy(accountData, copyData); err == nil { + account.DateModified = time.Now() + mngr.storage.AccountUpdated(account) mngr.writeAllAccounts() } else { @@ -160,6 +180,24 @@ func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) err return nil } +// AuthorizeAccount sets the authorization status of the account identified by the account email; if no such account exists, an error is returned. +func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool) error { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + account := mngr.findAccountByEmail(accountData.Email) + if account == nil { + return errors.Errorf("no account with the specified email exists") + } + + account.Data.Authorized = authorized + + mngr.storage.AccountUpdated(account) + mngr.writeAllAccounts() + + return nil +} + // RemoveAccount removes the account identified by the account email; if no such account exists, an error is returned. func (mngr *Manager) RemoveAccount(accountData *data.Account) error { mngr.mutex.Lock() diff --git a/internal/http/services/accounts/panel/panel.go b/internal/http/services/accounts/panel/panel.go new file mode 100644 index 0000000000..908f84ab7f --- /dev/null +++ b/internal/http/services/accounts/panel/panel.go @@ -0,0 +1,81 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package panel + +import ( + "html/template" + "net/http" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/internal/http/services/accounts/config" + "github.com/cs3org/reva/internal/http/services/accounts/data" +) + +// Panel represents the web interface panel of the accounts service. +type Panel struct { + conf *config.Configuration + log *zerolog.Logger + + tpl *template.Template +} + +func (panel *Panel) initialize(conf *config.Configuration, log *zerolog.Logger) error { + if conf == nil { + return errors.Errorf("no configuration provided") + } + panel.conf = conf + + if log == nil { + return errors.Errorf("no logger provided") + } + panel.log = log + + // Create the panel template + panel.tpl = template.New("panel") + if _, err := panel.tpl.Parse(panelTemplate); err != nil { + return errors.Wrap(err, "error while parsing panel template") + } + + return nil +} + +func (panel *Panel) Execute(w http.ResponseWriter, accounts *data.Accounts) error { + type TemplateData struct { + Count int + Accounts *data.Accounts + } + + data := TemplateData{ + Count: len(*accounts), + Accounts: accounts, + } + + return panel.tpl.Execute(w, data) +} + +// NewPanel creates a new web interface panel. +func NewPanel(conf *config.Configuration, log *zerolog.Logger) (*Panel, error) { + panel := &Panel{} + if err := panel.initialize(conf, log); err != nil { + return nil, errors.Wrapf(err, "unable to initialize the panel") + } + return panel, nil +} diff --git a/internal/http/services/accounts/panel/template.go b/internal/http/services/accounts/panel/template.go new file mode 100644 index 0000000000..c07af578e1 --- /dev/null +++ b/internal/http/services/accounts/panel/template.go @@ -0,0 +1,106 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package panel + +const panelTemplate = ` + + + + + + Accounts panel + + + +

Accounts ({{.Count}})

+

+

    + {{range .Accounts}} +
  • +

    + {{.Email}}
    + {{.FirstName}} {{.LastName}} (Joined: {{.DateCreated.Format "Jan 01, 2000 10:00"}}; Last modified: {{.DateModified.Format "Jan 01, 2000 10:00"}}) +

    +

    + API-Key: + {{if .Data.APIKey}} + {{.Data.APIKey}} + {{else}} + Not assigned + {{end}} +
    + Authorized: + {{if .Data.Authorized}} + Yes + {{else}} + No + {{end}} +

    +

    +

    + + + {{if .Data.Authorized}} + + {{else}} + + {{end}} + +   + +
    +

    +
    +
    • + {{end}} +
+

+ + + +` From 92dd65a8633cb80c6cf1b0f143777c1aafd74491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 17 Feb 2021 16:50:39 +0100 Subject: [PATCH 24/59] Add API key generator and endpoints --- internal/http/services/accounts/accounts.go | 40 ++++++++++++++- .../services/accounts/config/endpoints.go | 7 ++- internal/http/services/accounts/manager.go | 25 ++++++++++ .../http/services/accounts/panel/template.go | 4 +- pkg/apikey/apikey.go | 50 +++++++++++++++++++ 5 files changed, 121 insertions(+), 5 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 03271c273e..87dee03433 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -32,6 +32,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/pkg/apikey" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -106,11 +107,13 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { } endpoints := []Endpoint{ + {config.EndpointGenerateAPIKey, http.MethodGet, s.handleGenerateAPIKey}, {config.EndpointList, http.MethodGet, s.handleList}, {config.EndpointCreate, http.MethodPost, s.handleCreate}, {config.EndpointUpdate, http.MethodPost, s.handleUpdate}, {config.EndpointRemove, http.MethodPost, s.handleRemove}, {config.EndpointAuthorize, http.MethodPost, s.handleAuthorize}, + {config.EndpointAssignAPIKey, http.MethodPost, s.handleAssignAPIKey}, } // The default response is an unknown endpoint (for the specified method) @@ -148,6 +151,39 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(jsonData) } +func (s *svc) handleGenerateAPIKey(values url.Values, body []byte) (interface{}, error) { + email := values.Get("email") + flags := apikey.FlagDefault + + if strings.EqualFold(values.Get("isScienceMesh"), "true") { + flags |= apikey.FlagScienceMesh + } + + if len(email) == 0 { + return nil, errors.Errorf("no email provided") + } + + apiKey, err := apikey.GenerateAPIKey(email, flags) + if err != nil { + return nil, errors.Wrap(err, "unable to generate API key") + } + return map[string]string{"apiKey": apiKey}, nil +} + +func (s *svc) handleAssignAPIKey(values url.Values, body []byte) (interface{}, error) { + account, err := s.unmarshalRequestData(body) + if err != nil { + return nil, err + } + + // Assign a new API key to the account through the account manager + if err := s.manager.AssignAPIKeyToAccount(account, apikey.FlagDefault); err != nil { + return nil, errors.Wrap(err, "unable to assign API key") + } + + return nil, nil +} + func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { return s.manager.CloneAccounts(), nil } @@ -200,9 +236,9 @@ func (s *svc) handleAuthorize(values url.Values, body []byte) (interface{}, erro return nil, err } - if val, ok := values["status"]; ok && len(val) > 0 { + if val := values.Get("status"); len(val) > 0 { var authorize bool - switch strings.ToLower(val[0]) { + switch strings.ToLower(val) { case "true": authorize = true diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index f6077ca6c7..dd5cedbdcc 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -19,9 +19,12 @@ package config const ( - // EndpointPanel is the endpoint of the web interface panel. + // EndpointPanel is the endpoint path of the web interface panel. EndpointPanel = "/panel" + // EndpointGenerateAPIKey is the endpoint path of the API key generator. + EndpointGenerateAPIKey = "/generate-api-key" + // EndpointList is the endpoint path for listing all stored accounts. EndpointList = "/list" @@ -34,4 +37,6 @@ const ( // EndpointAuthorize is the endpoint path for account authorization EndpointAuthorize = "/authorize" + // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. + EndpointAssignAPIKey = "/assign-api-key" ) diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 0dcf4704b2..26825fb906 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -198,6 +198,31 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool return nil } +func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int16) error { + mngr.mutex.Lock() + defer mngr.mutex.Unlock() + + account := mngr.findAccountByEmail(accountData.Email) + if account == nil { + return errors.Errorf("no account with the specified email exists") + } + + if len(account.Data.APIKey) > 0 { + return errors.Errorf("the account already has an API key assigned") + } + + apiKey, err := apikey.GenerateAPIKey(strings.ToLower(account.Email), flags) // Use the (lowered) email address as the key's salt value + if err != nil { + return errors.Wrap(err, "error while generating API key") + } + account.Data.APIKey = apiKey + + mngr.storage.AccountUpdated(account) + mngr.writeAllAccounts() + + return nil +} + // RemoveAccount removes the account identified by the account email; if no such account exists, an error is returned. func (mngr *Manager) RemoveAccount(accountData *data.Account) error { mngr.mutex.Lock() diff --git a/internal/http/services/accounts/panel/template.go b/internal/http/services/accounts/panel/template.go index c07af578e1..0e17591452 100644 --- a/internal/http/services/accounts/panel/template.go +++ b/internal/http/services/accounts/panel/template.go @@ -67,7 +67,7 @@ const panelTemplate = ` {{.FirstName}} {{.LastName}} (Joined: {{.DateCreated.Format "Jan 01, 2000 10:00"}}; Last modified: {{.DateModified.Format "Jan 01, 2000 10:00"}})

- API-Key: + API Key: {{if .Data.APIKey}} {{.Data.APIKey}} {{else}} @@ -83,7 +83,7 @@ const panelTemplate = `

- + {{if .Data.Authorized}} diff --git a/pkg/apikey/apikey.go b/pkg/apikey/apikey.go index 2727ab15c8..a0fcf075c5 100644 --- a/pkg/apikey/apikey.go +++ b/pkg/apikey/apikey.go @@ -18,5 +18,55 @@ package apikey +import ( + "crypto/md5" + "crypto/rand" + "fmt" + + "github.com/pkg/errors" +) + // APIKey is the type used to store API keys. type APIKey = string + +const ( + // FlagDefault marks API keys for default (community) accounts. + FlagDefault int16 = 0x0000 + // FlagScienceMesh marks API keys for ScienceMesh (partner) accounts. + FlagScienceMesh int16 = 0x0001 +) + +const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + +// GenerateAPIKey generates a new (random) API key which also contains flags and a (salted) hash. +// An API key has the following format: +// +func GenerateAPIKey(salt string, flags int16) (APIKey, error) { + randomString, err := generateRandomString(30) + if err != nil { + return "", errors.Wrap(err, "unable to generate API key") + } + + // To verify an API key, a hash is used which contains, beside the random string and flags, the email address + hash := md5.New() + hash.Write([]byte(randomString)) + hash.Write([]byte(salt)) + hash.Write([]byte(fmt.Sprintf("%04x", flags))) + + return fmt.Sprintf("%s%02x%032x", randomString, flags, hash.Sum(nil)), nil +} + +func generateRandomString(n int) (string, error) { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return "", err + } + + str := "" + for _, v := range b { + str += string(charset[int(v)%len(charset)]) + } + + return str, nil +} From 17a7b46afe3fc2a3cff32f1d0c05c745ca140f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 18 Feb 2021 14:21:46 +0100 Subject: [PATCH 25/59] Add query endpoints --- internal/http/services/accounts/accounts.go | 33 ++++++++++++++ .../services/accounts/config/endpoints.go | 4 ++ internal/http/services/accounts/manager.go | 45 +++++++++++++++++-- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 87dee03433..5bfdd257b9 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -109,10 +109,12 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { endpoints := []Endpoint{ {config.EndpointGenerateAPIKey, http.MethodGet, s.handleGenerateAPIKey}, {config.EndpointList, http.MethodGet, s.handleList}, + {config.EndpointFind, http.MethodGet, s.handleFind}, {config.EndpointCreate, http.MethodPost, s.handleCreate}, {config.EndpointUpdate, http.MethodPost, s.handleUpdate}, {config.EndpointRemove, http.MethodPost, s.handleRemove}, {config.EndpointAuthorize, http.MethodPost, s.handleAuthorize}, + {config.EndpointIsAuthorized, http.MethodGet, s.handleIsAuthorized}, {config.EndpointAssignAPIKey, http.MethodPost, s.handleAssignAPIKey}, } @@ -188,6 +190,22 @@ func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { return s.manager.CloneAccounts(), nil } +func (s *svc) handleFind(values url.Values, body []byte) (interface{}, error) { + by := values.Get("by") + value := values.Get("value") + + if len(by) == 0 && len(value) == 0 { + return nil, errors.Errorf("missing search criteria") + } + + // Find the account using the account manager + account, err := s.manager.FindAccount(by, value) + if err != nil { + return nil, errors.Wrap(err, "no user found") + } + return map[string]interface{}{"account": account}, nil +} + func (s *svc) handleCreate(values url.Values, body []byte) (interface{}, error) { account, err := s.unmarshalRequestData(body) if err != nil { @@ -230,6 +248,21 @@ func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) return nil, nil } +func (s *svc) handleIsAuthorized(values url.Values, body []byte) (interface{}, error) { + apiKey := values.Get("apiKey") + + if len(apiKey) == 0 { + return nil, errors.Errorf("no API key specified") + } + + // Find the account associated with the given API key + account, err := s.manager.FindAccount(FindByAPIKey, apiKey) + if err != nil { + return nil, errors.Wrap(err, "no user with the specified API key found") + } + return account.Data.Authorized, nil +} + func (s *svc) handleAuthorize(values url.Values, body []byte) (interface{}, error) { account, err := s.unmarshalRequestData(body) if err != nil { diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index dd5cedbdcc..65cb00163e 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -27,6 +27,8 @@ const ( // EndpointList is the endpoint path for listing all stored accounts. EndpointList = "/list" + // EndpointFind is the endpoint path for finding accounts. + EndpointFind = "/find" // EndpointCreate is the endpoint path for account creation. EndpointCreate = "/create" @@ -37,6 +39,8 @@ const ( // EndpointAuthorize is the endpoint path for account authorization EndpointAuthorize = "/authorize" + // EndpointIsAuthorized is the endpoint path used to check the authorization status of an account. + EndpointIsAuthorized = "/is-authorized" // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. EndpointAssignAPIKey = "/assign-api-key" ) diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 26825fb906..33073016cc 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -35,6 +35,11 @@ import ( "github.com/cs3org/reva/pkg/apikey" ) +const ( + FindByEmail = "email" + FindByAPIKey = "apikey" +) + // Manager is responsible for all user account related tasks. type Manager struct { conf *config.Configuration @@ -180,6 +185,31 @@ func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) err return nil } +func (mngr *Manager) FindAccount(by string, value string) (*data.Account, error) { + mngr.mutex.RLock() + defer mngr.mutex.RUnlock() + + var account *data.Account + switch strings.ToLower(by) { + case FindByEmail: + account = mngr.findAccountByEmail(value) + + case FindByAPIKey: + account = mngr.findAccountByAPIKey(value) + + default: + return nil, errors.Errorf("invalid search type %v", by) + } + + if account != nil { + // Clone the account to avoid external data changes + clonedAccount := *account + return &clonedAccount, nil + } + + return nil, errors.Errorf("no user found matching the specified criteria") +} + // AuthorizeAccount sets the authorization status of the account identified by the account email; if no such account exists, an error is returned. func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool) error { mngr.mutex.Lock() @@ -211,11 +241,18 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int1 return errors.Errorf("the account already has an API key assigned") } - apiKey, err := apikey.GenerateAPIKey(strings.ToLower(account.Email), flags) // Use the (lowered) email address as the key's salt value - if err != nil { - return errors.Wrap(err, "error while generating API key") + for { + apiKey, err := apikey.GenerateAPIKey(strings.ToLower(account.Email), flags) // Use the (lowered) email address as the key's salt value + if err != nil { + return errors.Wrap(err, "error while generating API key") + } + + // See if the key already exists (super extremely unlikely); if so, generate a new one and try again + if mngr.findAccountByAPIKey(apiKey) == nil { + account.Data.APIKey = apiKey + break + } } - account.Data.APIKey = apiKey mngr.storage.AccountUpdated(account) mngr.writeAllAccounts() From 314a3c80ab520c09ba35857d84d8c0e059c5ba33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 18 Feb 2021 15:07:21 +0100 Subject: [PATCH 26/59] Add API key verification --- internal/http/services/accounts/accounts.go | 24 ++++++- .../services/accounts/config/endpoints.go | 6 +- internal/http/services/accounts/manager.go | 2 +- pkg/apikey/apikey.go | 66 ++++++++++++++++--- 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 5bfdd257b9..f3b58216e6 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -108,6 +108,8 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { endpoints := []Endpoint{ {config.EndpointGenerateAPIKey, http.MethodGet, s.handleGenerateAPIKey}, + {config.EndpointVerifyAPIKey, http.MethodGet, s.handleVerifyAPIKey}, + {config.EndpointAssignAPIKey, http.MethodPost, s.handleAssignAPIKey}, {config.EndpointList, http.MethodGet, s.handleList}, {config.EndpointFind, http.MethodGet, s.handleFind}, {config.EndpointCreate, http.MethodPost, s.handleCreate}, @@ -115,7 +117,6 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { {config.EndpointRemove, http.MethodPost, s.handleRemove}, {config.EndpointAuthorize, http.MethodPost, s.handleAuthorize}, {config.EndpointIsAuthorized, http.MethodGet, s.handleIsAuthorized}, - {config.EndpointAssignAPIKey, http.MethodPost, s.handleAssignAPIKey}, } // The default response is an unknown endpoint (for the specified method) @@ -165,13 +166,32 @@ func (s *svc) handleGenerateAPIKey(values url.Values, body []byte) (interface{}, return nil, errors.Errorf("no email provided") } - apiKey, err := apikey.GenerateAPIKey(email, flags) + apiKey, err := apikey.GenerateAPIKey(strings.ToLower(email), flags) if err != nil { return nil, errors.Wrap(err, "unable to generate API key") } return map[string]string{"apiKey": apiKey}, nil } +func (s *svc) handleVerifyAPIKey(values url.Values, body []byte) (interface{}, error) { + apiKey := values.Get("apiKey") + email := values.Get("email") + + if len(apiKey) == 0 { + return nil, errors.Errorf("no API key provided") + } + + if len(email) == 0 { + return nil, errors.Errorf("no email provided") + } + + err := apikey.VerifyAPIKey(apiKey, strings.ToLower(email)) + if err != nil { + return nil, errors.Wrap(err, "invalid API key") + } + return nil, nil +} + func (s *svc) handleAssignAPIKey(values url.Values, body []byte) (interface{}, error) { account, err := s.unmarshalRequestData(body) if err != nil { diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index 65cb00163e..2332d3a42e 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -24,6 +24,10 @@ const ( // EndpointGenerateAPIKey is the endpoint path of the API key generator. EndpointGenerateAPIKey = "/generate-api-key" + // EndpointGenerateAPIKey is the endpoint path for API key verification. + EndpointVerifyAPIKey = "/verify-api-key" + // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. + EndpointAssignAPIKey = "/assign-api-key" // EndpointList is the endpoint path for listing all stored accounts. EndpointList = "/list" @@ -41,6 +45,4 @@ const ( EndpointAuthorize = "/authorize" // EndpointIsAuthorized is the endpoint path used to check the authorization status of an account. EndpointIsAuthorized = "/is-authorized" - // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. - EndpointAssignAPIKey = "/assign-api-key" ) diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 33073016cc..ffb5570cb2 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -228,7 +228,7 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool return nil } -func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int16) error { +func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8) error { mngr.mutex.Lock() defer mngr.mutex.Unlock() diff --git a/pkg/apikey/apikey.go b/pkg/apikey/apikey.go index a0fcf075c5..51352d099a 100644 --- a/pkg/apikey/apikey.go +++ b/pkg/apikey/apikey.go @@ -22,6 +22,8 @@ import ( "crypto/md5" "crypto/rand" "fmt" + hashpkg "hash" + "strconv" "github.com/pkg/errors" ) @@ -31,31 +33,75 @@ type APIKey = string const ( // FlagDefault marks API keys for default (community) accounts. - FlagDefault int16 = 0x0000 + FlagDefault int8 = 0x0000 // FlagScienceMesh marks API keys for ScienceMesh (partner) accounts. - FlagScienceMesh int16 = 0x0001 + FlagScienceMesh int8 = 0x0001 ) -const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" +const ( + randomStringLength = 30 + apiKeyLength = randomStringLength + 2 + 32 + + charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" +) // GenerateAPIKey generates a new (random) API key which also contains flags and a (salted) hash. // An API key has the following format: // -func GenerateAPIKey(salt string, flags int16) (APIKey, error) { - randomString, err := generateRandomString(30) +func GenerateAPIKey(salt string, flags int8) (APIKey, error) { + if len(salt) == 0 { + return "", errors.Errorf("no salt specified") + } + + randomString, err := generateRandomString(randomStringLength) if err != nil { return "", errors.Wrap(err, "unable to generate API key") } // To verify an API key, a hash is used which contains, beside the random string and flags, the email address - hash := md5.New() - hash.Write([]byte(randomString)) - hash.Write([]byte(salt)) - hash.Write([]byte(fmt.Sprintf("%04x", flags))) - + hash := calculateHash(randomString, flags, salt) return fmt.Sprintf("%s%02x%032x", randomString, flags, hash.Sum(nil)), nil } +// VerifyAPIKey checks if the API key is valid given the specified salt value. +func VerifyAPIKey(apiKey APIKey, salt string) error { + randomString, flags, hash, err := SplitAPIKey(apiKey) + if err != nil { + return errors.Wrap(err, "error while extracting API key information") + } + + hashCalc := calculateHash(randomString, flags, salt) + if fmt.Sprintf("%032x", hashCalc.Sum(nil)) != hash { + return errors.Errorf("the API key is invalid") + } + + return nil +} + +// SplitAPIKey splits an API key into its pieces: RandomString, Flags and Hash. +func SplitAPIKey(apiKey APIKey) (string, int8, string, error) { + if len(apiKey) != apiKeyLength { + return "", 0, "", errors.Errorf("invalid API key length") + } + + randomString := apiKey[:randomStringLength] + flags, err := strconv.Atoi(apiKey[randomStringLength : randomStringLength+2]) + if err != nil { + return "", 0, "", errors.Errorf("invalid API key format") + } + hash := apiKey[randomStringLength+2:] + + return randomString, int8(flags), hash, nil +} + +func calculateHash(randomString string, flags int8, salt string) hashpkg.Hash { + hash := md5.New() + _, _ = hash.Write([]byte(randomString)) + _, _ = hash.Write([]byte(salt)) + _, _ = hash.Write([]byte(fmt.Sprintf("%04x", flags))) + return hash +} + func generateRandomString(n int) (string, error) { b := make([]byte, n) _, err := rand.Read(b) From 6528962e98c1b0dc4ec6b30f6f205431930789cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 18 Feb 2021 15:59:44 +0100 Subject: [PATCH 27/59] Add email notifications --- .../http/services/accounts/config/config.go | 4 ++ .../http/services/accounts/email/email.go | 63 +++++++++++++++++++ .../http/services/accounts/email/template.go | 51 +++++++++++++++ internal/http/services/accounts/manager.go | 17 +++++ 4 files changed, 135 insertions(+) create mode 100644 internal/http/services/accounts/email/email.go create mode 100644 internal/http/services/accounts/email/template.go diff --git a/internal/http/services/accounts/config/config.go b/internal/http/services/accounts/config/config.go index 7861701968..f890fcd7be 100644 --- a/internal/http/services/accounts/config/config.go +++ b/internal/http/services/accounts/config/config.go @@ -18,6 +18,8 @@ package config +import "github.com/cs3org/reva/pkg/smtpclient" + // Configuration holds the general service configuration. type Configuration struct { Prefix string `mapstructure:"prefix"` @@ -29,4 +31,6 @@ type Configuration struct { File string `mapstructure:"file"` } `mapstructure:"file"` } `mapstructure:"storage"` + + SMTP *smtpclient.SMTPCredentials `mapstructure:"smtp"` } diff --git a/internal/http/services/accounts/email/email.go b/internal/http/services/accounts/email/email.go new file mode 100644 index 0000000000..f2f5a9b04b --- /dev/null +++ b/internal/http/services/accounts/email/email.go @@ -0,0 +1,63 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package email + +import ( + "bytes" + "text/template" + + "github.com/pkg/errors" + + "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/pkg/smtpclient" +) + +// SendAccountCreated sends an email about account creation. +func SendAccountCreated(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { + return send(recipient, "ScienceMesh: User account created", accountCreatedTemplate, account, smtp) +} + +// SendAPIKeyAssigned sends an email about API key assignment. +func SendAPIKeyAssigned(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { + return send(recipient, "ScienceMesh: Your API key", apiKeyAssignedTemplate, account, smtp) +} + +// SendAccountAuthorized sends an email about account authorization. +func SendAccountAuthorized(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { + return send(recipient, "ScienceMesh: Site registration authorized", accountAuthorizedTemplate, account, smtp) +} + +func send(recipient string, subject string, bodyTemplate string, data interface{}, smtp *smtpclient.SMTPCredentials) error { + // Do not fail if no SMTP client or recipient is given + if smtp == nil || len(recipient) == 0 { + return nil + } + + tpl := template.New("email") + if _, err := tpl.Parse(bodyTemplate); err != nil { + return errors.Wrap(err, "error while parsing email template") + } + + var body bytes.Buffer + if err := tpl.Execute(&body, data); err != nil { + return errors.Wrap(err, "error while executing email template") + } + + return smtp.SendMail(recipient, subject, body.String()) +} diff --git a/internal/http/services/accounts/email/template.go b/internal/http/services/accounts/email/template.go new file mode 100644 index 0000000000..25df085bec --- /dev/null +++ b/internal/http/services/accounts/email/template.go @@ -0,0 +1,51 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package email + +const accountCreatedTemplate = ` +Dear {{.FirstName}} {{.LastName}}, + +Your ScienceMesh account has been successfully created! + +An administrator will soon create an API key for your account; you will receive a separate email containing the key. + +Kind regards, +The ScienceMesh Team +` + +const apiKeyAssignedTemplate = ` +Dear {{.FirstName}} {{.LastName}}, + +An API key has been created for your account: +{{.Data.APIKey}} + +Keep this key in a safe and secret place! + +Kind regards, +The ScienceMesh Team +` + +const accountAuthorizedTemplate = ` +Dear {{.FirstName}} {{.LastName}}, + +Congratulations - your site registration has been authorized! + +Kind regards, +The ScienceMesh Team +` diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index ffb5570cb2..1c4b052f41 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -31,8 +31,10 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/internal/http/services/accounts/email" "github.com/cs3org/reva/internal/http/services/accounts/panel" "github.com/cs3org/reva/pkg/apikey" + "github.com/cs3org/reva/pkg/smtpclient" ) const ( @@ -49,6 +51,7 @@ type Manager struct { storage data.Storage panel *panel.Panel + smtp *smtpclient.SMTPCredentials mutex sync.RWMutex } @@ -81,6 +84,11 @@ func (mngr *Manager) initialize(conf *config.Configuration, log *zerolog.Logger) return errors.Wrap(err, "unable to create panel") } + // Create the SMTP client + if conf.SMTP != nil { + mngr.smtp = smtpclient.NewSMTPCredentials(conf.SMTP) + } + return nil } @@ -156,6 +164,8 @@ func (mngr *Manager) CreateAccount(accountData *data.Account) error { mngr.accounts = append(mngr.accounts, account) mngr.storage.AccountAdded(account) mngr.writeAllAccounts() + + _ = email.SendAccountCreated(account, account.Email, mngr.smtp) } else { return errors.Wrap(err, "error while creating account") } @@ -220,11 +230,16 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool return errors.Errorf("no account with the specified email exists") } + authorizedOld := account.Data.Authorized account.Data.Authorized = authorized mngr.storage.AccountUpdated(account) mngr.writeAllAccounts() + if account.Data.Authorized && account.Data.Authorized != authorizedOld { + _ = email.SendAccountAuthorized(account, account.Email, mngr.smtp) + } + return nil } @@ -257,6 +272,8 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8 mngr.storage.AccountUpdated(account) mngr.writeAllAccounts() + _ = email.SendAPIKeyAssigned(account, account.Email, mngr.smtp) + return nil } From f0981c252b57159b60139a5510de24a79fcd89f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 18 Feb 2021 16:04:26 +0100 Subject: [PATCH 28/59] Panel date fixes --- internal/http/services/accounts/panel/template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/accounts/panel/template.go b/internal/http/services/accounts/panel/template.go index 0e17591452..c80b18eb4b 100644 --- a/internal/http/services/accounts/panel/template.go +++ b/internal/http/services/accounts/panel/template.go @@ -64,7 +64,7 @@ const panelTemplate = `

  • {{.Email}}
    - {{.FirstName}} {{.LastName}} (Joined: {{.DateCreated.Format "Jan 01, 2000 10:00"}}; Last modified: {{.DateModified.Format "Jan 01, 2000 10:00"}}) + {{.FirstName}} {{.LastName}} (Joined: {{.DateCreated.Format "Jan 02, 2006 15:04"}}; Last modified: {{.DateModified.Format "Jan 02, 2006 15:04"}})

    API Key: From 8739f98a897fd0e52309b80b9925802ff58c3057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 19 Feb 2021 12:49:05 +0100 Subject: [PATCH 29/59] Add notifications email option --- internal/http/services/accounts/accounts.go | 2 +- .../http/services/accounts/config/config.go | 3 +- internal/http/services/accounts/data/data.go | 2 +- .../http/services/accounts/email/email.go | 28 +++++++++++++------ internal/http/services/accounts/manager.go | 8 +++--- pkg/{ => mentix}/apikey/apikey.go | 0 6 files changed, 27 insertions(+), 16 deletions(-) rename pkg/{ => mentix}/apikey/apikey.go (100%) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index f3b58216e6..a8f48260ce 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -32,7 +32,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" - "github.com/cs3org/reva/pkg/apikey" + "github.com/cs3org/reva/pkg/mentix/apikey" "github.com/cs3org/reva/pkg/rhttp/global" ) diff --git a/internal/http/services/accounts/config/config.go b/internal/http/services/accounts/config/config.go index f890fcd7be..6a4a282183 100644 --- a/internal/http/services/accounts/config/config.go +++ b/internal/http/services/accounts/config/config.go @@ -32,5 +32,6 @@ type Configuration struct { } `mapstructure:"file"` } `mapstructure:"storage"` - SMTP *smtpclient.SMTPCredentials `mapstructure:"smtp"` + SMTP *smtpclient.SMTPCredentials `mapstructure:"smtp"` + NotificationsMail string `mapstructure:"notifications_mail"` } diff --git a/internal/http/services/accounts/data/data.go b/internal/http/services/accounts/data/data.go index 9f6b5b5639..78da401f9f 100644 --- a/internal/http/services/accounts/data/data.go +++ b/internal/http/services/accounts/data/data.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" - "github.com/cs3org/reva/pkg/apikey" + "github.com/cs3org/reva/pkg/mentix/apikey" "github.com/cs3org/reva/pkg/utils" ) diff --git a/internal/http/services/accounts/email/email.go b/internal/http/services/accounts/email/email.go index f2f5a9b04b..7fec66a082 100644 --- a/internal/http/services/accounts/email/email.go +++ b/internal/http/services/accounts/email/email.go @@ -29,23 +29,23 @@ import ( ) // SendAccountCreated sends an email about account creation. -func SendAccountCreated(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { - return send(recipient, "ScienceMesh: User account created", accountCreatedTemplate, account, smtp) +func SendAccountCreated(account *data.Account, recipients []string, smtp *smtpclient.SMTPCredentials) error { + return send(recipients, "ScienceMesh: User account created", accountCreatedTemplate, account, smtp) } // SendAPIKeyAssigned sends an email about API key assignment. -func SendAPIKeyAssigned(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { - return send(recipient, "ScienceMesh: Your API key", apiKeyAssignedTemplate, account, smtp) +func SendAPIKeyAssigned(account *data.Account, recipients []string, smtp *smtpclient.SMTPCredentials) error { + return send(recipients, "ScienceMesh: Your API key", apiKeyAssignedTemplate, account, smtp) } // SendAccountAuthorized sends an email about account authorization. -func SendAccountAuthorized(account *data.Account, recipient string, smtp *smtpclient.SMTPCredentials) error { - return send(recipient, "ScienceMesh: Site registration authorized", accountAuthorizedTemplate, account, smtp) +func SendAccountAuthorized(account *data.Account, recipients []string, smtp *smtpclient.SMTPCredentials) error { + return send(recipients, "ScienceMesh: Site registration authorized", accountAuthorizedTemplate, account, smtp) } -func send(recipient string, subject string, bodyTemplate string, data interface{}, smtp *smtpclient.SMTPCredentials) error { +func send(recipients []string, subject string, bodyTemplate string, data interface{}, smtp *smtpclient.SMTPCredentials) error { // Do not fail if no SMTP client or recipient is given - if smtp == nil || len(recipient) == 0 { + if smtp == nil { return nil } @@ -59,5 +59,15 @@ func send(recipient string, subject string, bodyTemplate string, data interface{ return errors.Wrap(err, "error while executing email template") } - return smtp.SendMail(recipient, subject, body.String()) + for _, recipient := range recipients { + if len(recipient) == 0 { + continue + } + + if err := smtp.SendMail(recipient, subject, body.String()); err != nil { + return errors.Wrapf(err, "failed sending email to %v", recipient) + } + } + + return nil } diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 1c4b052f41..8cc68c165c 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -33,7 +33,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/data" "github.com/cs3org/reva/internal/http/services/accounts/email" "github.com/cs3org/reva/internal/http/services/accounts/panel" - "github.com/cs3org/reva/pkg/apikey" + "github.com/cs3org/reva/pkg/mentix/apikey" "github.com/cs3org/reva/pkg/smtpclient" ) @@ -165,7 +165,7 @@ func (mngr *Manager) CreateAccount(accountData *data.Account) error { mngr.storage.AccountAdded(account) mngr.writeAllAccounts() - _ = email.SendAccountCreated(account, account.Email, mngr.smtp) + _ = email.SendAccountCreated(account, []string{account.Email, mngr.conf.NotificationsMail}, mngr.smtp) } else { return errors.Wrap(err, "error while creating account") } @@ -237,7 +237,7 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool mngr.writeAllAccounts() if account.Data.Authorized && account.Data.Authorized != authorizedOld { - _ = email.SendAccountAuthorized(account, account.Email, mngr.smtp) + _ = email.SendAccountAuthorized(account, []string{account.Email, mngr.conf.NotificationsMail}, mngr.smtp) } return nil @@ -272,7 +272,7 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8 mngr.storage.AccountUpdated(account) mngr.writeAllAccounts() - _ = email.SendAPIKeyAssigned(account, account.Email, mngr.smtp) + _ = email.SendAPIKeyAssigned(account, []string{account.Email, mngr.conf.NotificationsMail}, mngr.smtp) return nil } diff --git a/pkg/apikey/apikey.go b/pkg/mentix/apikey/apikey.go similarity index 100% rename from pkg/apikey/apikey.go rename to pkg/mentix/apikey/apikey.go From 880835c21e26f9d90ad1ac0af0856c07bf18c22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 19 Feb 2021 13:14:16 +0100 Subject: [PATCH 30/59] Create output directory for file storage --- internal/http/services/accounts/data/filestorage.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/http/services/accounts/data/filestorage.go b/internal/http/services/accounts/data/filestorage.go index 681e133758..fb9ec17c80 100644 --- a/internal/http/services/accounts/data/filestorage.go +++ b/internal/http/services/accounts/data/filestorage.go @@ -21,6 +21,8 @@ package data import ( "encoding/json" "io/ioutil" + "os" + "path/filepath" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -54,6 +56,10 @@ func (storage *FileStorage) initialize(conf *config.Configuration, log *zerolog. } storage.filePath = conf.Storage.File.File + // Create the file directory if necessary + dir := filepath.Dir(storage.filePath) + _ = os.MkdirAll(dir, 0755) + return nil } From fbb8706e16462f51fa8f17ac406fcb06432da759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 19 Feb 2021 14:12:48 +0100 Subject: [PATCH 31/59] Remove testing code --- internal/http/services/accounts/accounts.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index a8f48260ce..fa54494604 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -63,8 +63,6 @@ func (s *svc) Prefix() string { // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { - // TODO: For testing only - return []string{"/"} // This service currently only has one public endpoint (called "register") used for account registration return []string{config.EndpointCreate} } From 4e15bd8411d446bc9c273cce59d4554abc2bfa7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 22 Feb 2021 13:34:19 +0100 Subject: [PATCH 32/59] Add site ID generation --- internal/http/services/accounts/accounts.go | 17 +++++--- .../accounts/data/{data.go => account.go} | 14 +++++-- .../http/services/accounts/email/email.go | 7 ++-- internal/http/services/accounts/manager.go | 6 +-- .../http/services/accounts/panel/panel.go | 2 - .../http/services/accounts/panel/template.go | 9 ++-- pkg/mentix/{apikey => key}/apikey.go | 2 +- pkg/mentix/key/siteid.go | 42 +++++++++++++++++++ 8 files changed, 78 insertions(+), 21 deletions(-) rename internal/http/services/accounts/data/{data.go => account.go} (90%) rename pkg/mentix/{apikey => key}/apikey.go (99%) create mode 100644 pkg/mentix/key/siteid.go diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index fa54494604..89796a3e7a 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -32,7 +32,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/config" "github.com/cs3org/reva/internal/http/services/accounts/data" - "github.com/cs3org/reva/pkg/mentix/apikey" + "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -154,17 +154,17 @@ func (s *svc) handleRequestEndpoints(w http.ResponseWriter, r *http.Request) { func (s *svc) handleGenerateAPIKey(values url.Values, body []byte) (interface{}, error) { email := values.Get("email") - flags := apikey.FlagDefault + flags := key.FlagDefault if strings.EqualFold(values.Get("isScienceMesh"), "true") { - flags |= apikey.FlagScienceMesh + flags |= key.FlagScienceMesh } if len(email) == 0 { return nil, errors.Errorf("no email provided") } - apiKey, err := apikey.GenerateAPIKey(strings.ToLower(email), flags) + apiKey, err := key.GenerateAPIKey(strings.ToLower(email), flags) if err != nil { return nil, errors.Wrap(err, "unable to generate API key") } @@ -183,7 +183,7 @@ func (s *svc) handleVerifyAPIKey(values url.Values, body []byte) (interface{}, e return nil, errors.Errorf("no email provided") } - err := apikey.VerifyAPIKey(apiKey, strings.ToLower(email)) + err := key.VerifyAPIKey(apiKey, strings.ToLower(email)) if err != nil { return nil, errors.Wrap(err, "invalid API key") } @@ -196,8 +196,13 @@ func (s *svc) handleAssignAPIKey(values url.Values, body []byte) (interface{}, e return nil, err } + flags := key.FlagDefault + if _, ok := values["isScienceMesh"]; ok { + flags |= key.FlagScienceMesh + } + // Assign a new API key to the account through the account manager - if err := s.manager.AssignAPIKeyToAccount(account, apikey.FlagDefault); err != nil { + if err := s.manager.AssignAPIKeyToAccount(account, flags); err != nil { return nil, errors.Wrap(err, "unable to assign API key") } diff --git a/internal/http/services/accounts/data/data.go b/internal/http/services/accounts/data/account.go similarity index 90% rename from internal/http/services/accounts/data/data.go rename to internal/http/services/accounts/data/account.go index 78da401f9f..47ff80a0ea 100644 --- a/internal/http/services/accounts/data/data.go +++ b/internal/http/services/accounts/data/account.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" - "github.com/cs3org/reva/pkg/mentix/apikey" + "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/utils" ) @@ -41,13 +41,21 @@ type Account struct { // AccountData holds additional data for a user account. type AccountData struct { - APIKey apikey.APIKey `json:"apiKey"` - Authorized bool `json:"authorized"` + APIKey key.APIKey `json:"apiKey"` + Authorized bool `json:"authorized"` } // Accounts holds an array of user accounts. type Accounts = []*Account +func (acc *Account) GetSiteID() key.SiteIdentifier { + if id, err := key.CalculateSiteID(acc.Data.APIKey); err == nil { + return id + } + + return "" +} + // Copy copies the data of the given account to this account; if copyData is true, the account data is copied as well. func (acc *Account) Copy(other *Account, copyData bool) error { if err := other.verify(); err != nil { diff --git a/internal/http/services/accounts/email/email.go b/internal/http/services/accounts/email/email.go index 7fec66a082..009048bc23 100644 --- a/internal/http/services/accounts/email/email.go +++ b/internal/http/services/accounts/email/email.go @@ -64,9 +64,10 @@ func send(recipients []string, subject string, bodyTemplate string, data interfa continue } - if err := smtp.SendMail(recipient, subject, body.String()); err != nil { - return errors.Wrapf(err, "failed sending email to %v", recipient) - } + // Send the mail w/o blocking the main thread + go func(recipient string) { + _ = smtp.SendMail(recipient, subject, body.String()) + }(recipient) } return nil diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 8cc68c165c..3175f05839 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -33,7 +33,7 @@ import ( "github.com/cs3org/reva/internal/http/services/accounts/data" "github.com/cs3org/reva/internal/http/services/accounts/email" "github.com/cs3org/reva/internal/http/services/accounts/panel" - "github.com/cs3org/reva/pkg/mentix/apikey" + "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/smtpclient" ) @@ -130,7 +130,7 @@ func (mngr *Manager) findAccountByEmail(email string) *data.Account { return nil } -func (mngr *Manager) findAccountByAPIKey(key apikey.APIKey) *data.Account { +func (mngr *Manager) findAccountByAPIKey(key key.APIKey) *data.Account { if key == "" { return nil } @@ -257,7 +257,7 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8 } for { - apiKey, err := apikey.GenerateAPIKey(strings.ToLower(account.Email), flags) // Use the (lowered) email address as the key's salt value + apiKey, err := key.GenerateAPIKey(strings.ToLower(account.Email), flags) // Use the (lowered) email address as the key's salt value if err != nil { return errors.Wrap(err, "error while generating API key") } diff --git a/internal/http/services/accounts/panel/panel.go b/internal/http/services/accounts/panel/panel.go index 908f84ab7f..9170039950 100644 --- a/internal/http/services/accounts/panel/panel.go +++ b/internal/http/services/accounts/panel/panel.go @@ -59,12 +59,10 @@ func (panel *Panel) initialize(conf *config.Configuration, log *zerolog.Logger) func (panel *Panel) Execute(w http.ResponseWriter, accounts *data.Accounts) error { type TemplateData struct { - Count int Accounts *data.Accounts } data := TemplateData{ - Count: len(*accounts), Accounts: accounts, } diff --git a/internal/http/services/accounts/panel/template.go b/internal/http/services/accounts/panel/template.go index c80b18eb4b..7493d15287 100644 --- a/internal/http/services/accounts/panel/template.go +++ b/internal/http/services/accounts/panel/template.go @@ -50,14 +50,14 @@ const panelTemplate = ` font-family: monospace !important; } button { - min-width: 130px; + min-width: 140px; } Accounts panel -

    Accounts ({{.Count}})

    +

    Accounts ({{.Accounts | len}})

      {{range .Accounts}} @@ -74,6 +74,8 @@ const panelTemplate = ` Not assigned {{end}}
      + Site ID: {{.GetSiteID}} +

      Authorized: {{if .Data.Authorized}} Yes @@ -83,7 +85,8 @@ const panelTemplate = `

      - + + {{if .Data.Authorized}} diff --git a/pkg/mentix/apikey/apikey.go b/pkg/mentix/key/apikey.go similarity index 99% rename from pkg/mentix/apikey/apikey.go rename to pkg/mentix/key/apikey.go index 51352d099a..3e924240fa 100644 --- a/pkg/mentix/apikey/apikey.go +++ b/pkg/mentix/key/apikey.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package apikey +package key import ( "crypto/md5" diff --git a/pkg/mentix/key/siteid.go b/pkg/mentix/key/siteid.go new file mode 100644 index 0000000000..bc8af06c98 --- /dev/null +++ b/pkg/mentix/key/siteid.go @@ -0,0 +1,42 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package key + +import ( + "fmt" + "hash/crc64" + + "github.com/pkg/errors" +) + +// SiteIdentifier is the type used to store site identifiers. +type SiteIdentifier = string + +// CalculateSiteID calculates a (stable) site ID from the given API key. +// The site ID is actually the CRC64 hash of the provided API key, thus it is stable for any given key. +func CalculateSiteID(apiKey APIKey) (SiteIdentifier, error) { + if len(apiKey) != apiKeyLength { + return "", errors.Errorf("invalid API key length") + } + + hash := crc64.New(crc64.MakeTable(crc64.ECMA)) + _, _ = hash.Write([]byte(apiKey)) + value := hash.Sum(nil) + return fmt.Sprintf("%4x-%4x-%4x-%4x", value[:2], value[2:4], value[4:6], value[6:]), nil +} From a613c66393b5b68927c55f864b519708fb1ba655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 12:44:40 +0100 Subject: [PATCH 33/59] Remove admin endpoints --- pkg/mentix/connectors/localfile.go | 22 ------ pkg/mentix/exchangers/importers/adminapi.go | 60 --------------- .../exchangers/importers/adminapi/query.go | 73 ------------------- .../exchangers/importers/reqimporter.go | 7 -- pkg/mentix/meshdata/meshdata.go | 44 +++++------ pkg/mentix/meshdata/site.go | 17 ----- 6 files changed, 18 insertions(+), 205 deletions(-) delete mode 100755 pkg/mentix/exchangers/importers/adminapi.go delete mode 100755 pkg/mentix/exchangers/importers/adminapi/query.go diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index 6c154adc2c..a3c57d5259 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -98,12 +98,6 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa case meshdata.StatusObsolete: err = connector.unmergeData(meshData, updatedData) - - case meshdata.StatusAuthorize: - err = connector.authorizeData(meshData, updatedData, true) - - case meshdata.StatusUnauthorize: - err = connector.authorizeData(meshData, updatedData, false) } if err != nil { @@ -144,22 +138,6 @@ func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, up return nil } -func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData, authorize bool) error { - for _, placeholderSite := range updatedData.Sites { - if site := meshData.FindSite(placeholderSite.ID); site != nil { - if authorize { - meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "true") - } else { - meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") - } - } else { - return fmt.Errorf("no site with id '%v' found", placeholderSite.Name) - } - } - - return nil -} - func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { for _, site := range meshData.Sites { site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites diff --git a/pkg/mentix/exchangers/importers/adminapi.go b/pkg/mentix/exchangers/importers/adminapi.go deleted file mode 100755 index 53ee50748b..0000000000 --- a/pkg/mentix/exchangers/importers/adminapi.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package importers - -import ( - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchangers/importers/adminapi" -) - -// AdminAPIImporter implements the administrative API importer. -type AdminAPIImporter struct { - BaseRequestImporter -} - -// Activate activates the importer. -func (importer *AdminAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { - return err - } - - // Store AdminAPI specifics - importer.SetEndpoint(conf.Importers.AdminAPI.Endpoint, conf.Importers.AdminAPI.IsProtected) - importer.SetEnabledConnectors(conf.Importers.AdminAPI.EnabledConnectors) - - importer.authorizeSiteActionHandler = adminapi.HandleAuthorizeSiteQuery - - return nil -} - -// GetID returns the ID of the importer. -func (importer *AdminAPIImporter) GetID() string { - return config.ImporterIDAdminAPI -} - -// GetName returns the display name of the importer. -func (importer *AdminAPIImporter) GetName() string { - return "AdminAPI" -} - -func init() { - registerImporter(&AdminAPIImporter{}) -} diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go deleted file mode 100755 index cb7a65a198..0000000000 --- a/pkg/mentix/exchangers/importers/adminapi/query.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package adminapi - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" - - "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/network" -) - -func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { - jsonData := make(map[string]interface{}) - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, err - } - - if value, ok := jsonData["id"]; ok { - if id, ok := value.(string); ok { - site := &meshdata.Site{} - site.ID = id // We only need to store the ID of the site - - meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - return meshData, nil - } - - return nil, fmt.Errorf("site id invalid") - } - - return nil, fmt.Errorf("site id missing") -} - -func handleAdminQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { - meshData, err := decodeAdminQueryData(data) - if err != nil { - return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil - } - meshData.Status = status - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].Name}), nil -} - -// HandleAuthorizeSiteQuery sets the authorization status of a site. -func HandleAuthorizeSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - status := params.Get("status") - - if strings.EqualFold(status, "true") { - return handleAdminQuery(data, params, meshdata.StatusAuthorize, "SITE_AUTHORIZED") - } else if strings.EqualFold(status, "false") { - return handleAdminQuery(data, params, meshdata.StatusUnauthorize, "SITE_UNAUTHORIZED") - } - - return nil, http.StatusBadRequest, network.CreateResponse("INVALID_QUERY", network.ResponseParams{}), nil -} diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 673e4ff161..fbe54e3c0c 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -32,7 +32,6 @@ import ( const ( queryActionRegisterSite = "register" queryActionUnregisterSite = "unregister" - queryActionAuthorizeSite = "authorize" ) type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) @@ -44,7 +43,6 @@ type BaseRequestImporter struct { registerSiteActionHandler queryCallback unregisterSiteActionHandler queryCallback - authorizeSiteActionHandler queryCallback } // HandleRequest handles the actual HTTP request. @@ -88,11 +86,6 @@ func (importer *BaseRequestImporter) handleQuery(data []byte, path string, param return importer.unregisterSiteActionHandler(data, params) } - case queryActionAuthorizeSite: - if importer.authorizeSiteActionHandler != nil { - return importer.authorizeSiteActionHandler(data, params) - } - default: return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 05289b72d1..7e6ec4e580 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -33,10 +33,6 @@ const ( // StatusObsolete flags the mesh data for removal. StatusObsolete - // StatusAuthorize flags the mesh data for authorization. - StatusAuthorize - // StatusUnauthorize flags the mesh data for unauthorization. - StatusUnauthorize ) // MeshData holds the entire mesh data managed by Mentix. @@ -65,16 +61,14 @@ func (meshData *MeshData) AddSite(site *Site) { } // RemoveSite removes the provided site. -func (meshData *MeshData) RemoveSite(id string) { - if site := meshData.FindSite(id); site != nil { - for idx, siteExisting := range meshData.Sites { - if siteExisting == site { - lastIdx := len(meshData.Sites) - 1 - meshData.Sites[idx] = meshData.Sites[lastIdx] - meshData.Sites[lastIdx] = nil - meshData.Sites = meshData.Sites[:lastIdx] - break - } +func (meshData *MeshData) RemoveSite(site *Site) { + for idx, siteExisting := range meshData.Sites { + if strings.EqualFold(siteExisting.ID, site.ID) { // Remove the site by its ID + lastIdx := len(meshData.Sites) - 1 + meshData.Sites[idx] = meshData.Sites[lastIdx] + meshData.Sites[lastIdx] = nil + meshData.Sites = meshData.Sites[:lastIdx] + break } } } @@ -99,16 +93,14 @@ func (meshData *MeshData) AddServiceType(serviceType *ServiceType) { } // RemoveServiceType removes the provided service type. -func (meshData *MeshData) RemoveServiceType(name string) { - if serviceType := meshData.FindServiceType(name); serviceType != nil { - for idx, svcTypeExisting := range meshData.ServiceTypes { - if svcTypeExisting == serviceType { - lastIdx := len(meshData.ServiceTypes) - 1 - meshData.ServiceTypes[idx] = meshData.ServiceTypes[lastIdx] - meshData.ServiceTypes[lastIdx] = nil - meshData.ServiceTypes = meshData.ServiceTypes[:lastIdx] - break - } +func (meshData *MeshData) RemoveServiceType(serviceType *ServiceType) { + for idx, svcTypeExisting := range meshData.ServiceTypes { + if strings.EqualFold(svcTypeExisting.Name, serviceType.Name) { // Remove the service type by its name + lastIdx := len(meshData.ServiceTypes) - 1 + meshData.ServiceTypes[idx] = meshData.ServiceTypes[lastIdx] + meshData.ServiceTypes[lastIdx] = nil + meshData.ServiceTypes = meshData.ServiceTypes[:lastIdx] + break } } } @@ -137,11 +129,11 @@ func (meshData *MeshData) Merge(inData *MeshData) { // Unmerge removes data from another MeshData instance from this one. func (meshData *MeshData) Unmerge(inData *MeshData) { for _, site := range inData.Sites { - meshData.RemoveSite(site.ID) + meshData.RemoveSite(site) } for _, serviceType := range inData.ServiceTypes { - meshData.RemoveServiceType(serviceType.Name) + meshData.RemoveServiceType(serviceType) } } diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index ce41174ad5..b3524795c4 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -123,29 +123,12 @@ func (site *Site) InferMissingData() { } } - // Automatically assign an ID to this site if it is missing - if len(site.ID) == 0 { - site.generateID() - } - // Infer missing for services for _, service := range site.Services { service.InferMissingData() } } -// Name, Domain -func (site *Site) generateID() { - host := site.Domain - if site.Homepage != "" { - if hostURL, err := url.Parse(site.Homepage); err == nil { - host = network.ExtractDomainFromURL(hostURL, true) - } - } - - site.ID = fmt.Sprintf("%s::[%s]", host, site.Name) -} - // GetSiteTypeName returns the readable name of the given site type. func GetSiteTypeName(siteType SiteType) string { switch siteType { From bf4a13a4f57c849217989c72288089c088f44ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 12:55:06 +0100 Subject: [PATCH 34/59] Site IDs are now stored in GOCDB --- pkg/mentix/connectors/gocdb.go | 6 ++++++ pkg/mentix/meshdata/properties.go | 2 ++ 2 files changed, 8 insertions(+) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 75742f27d3..5d5c50b3a7 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -128,6 +128,11 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { for _, site := range sites.Sites { properties := connector.extensionsToMap(&site.Extensions) + siteID := meshdata.GetPropertyValue(properties, meshdata.PropertySiteID, "") + if len(siteID) == 0 { + return fmt.Errorf("site ID missing for site '%v'", site.ShortName) + } + // Sites coming from the GOCDB are always authorized by default if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 { meshdata.SetPropertyValue(&properties, meshdata.PropertyAuthorized, "true") @@ -138,6 +143,7 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { meshsite := &meshdata.Site{ Type: meshdata.SiteTypeScienceMesh, // All sites stored in the GOCDB are part of the mesh + ID: siteID, Name: site.ShortName, FullName: site.OfficialName, Organization: organization, diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 0374ec2e2f..3538c83198 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,6 +21,8 @@ package meshdata import "strings" const ( + // PropertySiteID identifies the site ID property. + PropertySiteID = "site_id" // PropertyAuthorized identifies the authorization status property. PropertyAuthorized = "authorized" // PropertyOrganization identifies the organization property. From 55a1201d294765d7a8b1dfa0c62f30592fe209d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 13:00:17 +0100 Subject: [PATCH 35/59] Cleanup --- internal/http/services/mentix/mentix.go | 4 ---- pkg/mentix/config/config.go | 6 ------ pkg/mentix/config/ids.go | 2 -- 3 files changed, 12 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index f07070d31d..b33a934348 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -141,10 +141,6 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Importers.WebAPI.Endpoint = "/sites" } - if conf.Importers.AdminAPI.Endpoint == "" { - conf.Importers.AdminAPI.Endpoint = "/admin" - } - // Exporters addDefaultConnector := func(enabledList *[]string) { if len(*enabledList) == 0 { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index b84ded160f..806c1015f9 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -41,12 +41,6 @@ type Configuration struct { IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` - - AdminAPI struct { - Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` - EnabledConnectors []string `mapstructure:"enabled_connectors"` - } `mapstructure:"adminapi"` } `mapstructure:"importers"` Exporters struct { diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 9fbb020693..04ff545ec3 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,8 +28,6 @@ const ( const ( // ImporterIDWebAPI is the identifier for the WebAPI importer. ImporterIDWebAPI = "webapi" - // ImporterIDAdminAPI is the identifier for the AdminAPI importer. - ImporterIDAdminAPI = "adminapi" ) const ( From 46ffcefa56c15439c99c62d95254aba0877a9e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 13:33:40 +0100 Subject: [PATCH 36/59] Streamline request exchangers --- pkg/mentix/exchangers/exporters/cs3api.go | 2 +- .../exchangers/exporters/cs3api/query.go | 8 ++-- .../exchangers/exporters/reqexporter.go | 31 +++----------- .../exchangers/exporters/siteloc/query.go | 8 ++-- .../exchangers/exporters/sitelocations.go | 2 +- pkg/mentix/exchangers/exporters/webapi.go | 2 +- .../exchangers/exporters/webapi/query.go | 6 +-- .../exchangers/importers/reqimporter.go | 42 ++++--------------- pkg/mentix/exchangers/importers/webapi.go | 4 +- .../exchangers/importers/webapi/query.go | 4 +- pkg/mentix/exchangers/reqexchanger.go | 28 +++++++++++++ 11 files changed, 60 insertions(+), 77 deletions(-) diff --git a/pkg/mentix/exchangers/exporters/cs3api.go b/pkg/mentix/exchangers/exporters/cs3api.go index 3102c0ce88..983c661070 100755 --- a/pkg/mentix/exchangers/exporters/cs3api.go +++ b/pkg/mentix/exchangers/exporters/cs3api.go @@ -40,7 +40,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint, conf.Exporters.CS3API.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors) - exporter.defaultActionHandler = cs3api.HandleDefaultQuery + exporter.RegisterActionHandler("", cs3api.HandleDefaultQuery) return nil } diff --git a/pkg/mentix/exchangers/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go index a107a1cce8..9859953bfc 100755 --- a/pkg/mentix/exchangers/exporters/cs3api/query.go +++ b/pkg/mentix/exchangers/exporters/cs3api/query.go @@ -30,20 +30,20 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { - return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) + return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) } // Marshal the OCM data as JSON data, err := json.MarshalIndent(ocmData, "", "\t") if err != nil { - return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) + return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) } - return http.StatusOK, data, nil + return nil, http.StatusOK, data, nil } func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) { diff --git a/pkg/mentix/exchangers/exporters/reqexporter.go b/pkg/mentix/exchangers/exporters/reqexporter.go index 368bb56260..42cb274b11 100644 --- a/pkg/mentix/exchangers/exporters/reqexporter.go +++ b/pkg/mentix/exchangers/exporters/reqexporter.go @@ -19,32 +19,23 @@ package exporters import ( - "fmt" + "io/ioutil" "net/http" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/exchangers" - "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryActionDefault = "" -) - -type queryCallback func(*meshdata.MeshData, url.Values) (int, []byte, error) - // BaseRequestExporter implements basic exporter functionality common to all request exporters. type BaseRequestExporter struct { BaseExporter exchangers.BaseRequestExchanger - - defaultActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { - status, respData, err := exporter.handleQuery(req.URL.Query()) + body, _ := ioutil.ReadAll(req.Body) + status, respData, err := exporter.handleQuery(body, req.URL.Query()) if err != nil { respData = []byte(err.Error()) } @@ -52,21 +43,11 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req _, _ = resp.Write(respData) } -func (exporter *BaseRequestExporter) handleQuery(params url.Values) (int, []byte, error) { +func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values) (int, []byte, error) { // Data is read, so lock it for writing exporter.Locker().RLock() defer exporter.Locker().RUnlock() - action := params.Get("action") - switch strings.ToLower(action) { - case queryActionDefault: - if exporter.defaultActionHandler != nil { - return exporter.defaultActionHandler(exporter.MeshData(), params) - } - - default: - return http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) - } - - return http.StatusNotImplemented, []byte{}, fmt.Errorf("unhandled query for action '%v'", action) + _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params) + return status, data, err } diff --git a/pkg/mentix/exchangers/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go index 491462d5e1..09aa4f985b 100755 --- a/pkg/mentix/exchangers/exporters/siteloc/query.go +++ b/pkg/mentix/exchangers/exporters/siteloc/query.go @@ -28,20 +28,20 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { // Convert the mesh data locData, err := convertMeshDataToLocationData(meshData) if err != nil { - return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) + return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) } // Marshal the location data as JSON data, err := json.MarshalIndent(locData, "", "\t") if err != nil { - return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) + return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) } - return http.StatusOK, data, nil + return nil, http.StatusOK, data, nil } func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { diff --git a/pkg/mentix/exchangers/exporters/sitelocations.go b/pkg/mentix/exchangers/exporters/sitelocations.go index ce6034f89b..d3f8f1b352 100755 --- a/pkg/mentix/exchangers/exporters/sitelocations.go +++ b/pkg/mentix/exchangers/exporters/sitelocations.go @@ -40,7 +40,7 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint, conf.Exporters.SiteLocations.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors) - exporter.defaultActionHandler = siteloc.HandleDefaultQuery + exporter.RegisterActionHandler("", siteloc.HandleDefaultQuery) return nil } diff --git a/pkg/mentix/exchangers/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go index f784c3df0d..cb7c59965f 100755 --- a/pkg/mentix/exchangers/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -42,7 +42,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo exporter.allowUnauthorizedSites = true - exporter.defaultActionHandler = webapi.HandleDefaultQuery + exporter.RegisterActionHandler("", webapi.HandleDefaultQuery) return nil } diff --git a/pkg/mentix/exchangers/exporters/webapi/query.go b/pkg/mentix/exchangers/exporters/webapi/query.go index 362968fecb..e4886b91bb 100755 --- a/pkg/mentix/exchangers/exporters/webapi/query.go +++ b/pkg/mentix/exchangers/exporters/webapi/query.go @@ -28,12 +28,12 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values) (int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { - return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) } - return http.StatusOK, data, nil + return nil, http.StatusOK, data, nil } diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index fbe54e3c0c..1604c857f1 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -19,39 +19,29 @@ package importers import ( - "fmt" "io/ioutil" "net/http" "net/url" - "strings" "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) -const ( - queryActionRegisterSite = "register" - queryActionUnregisterSite = "unregister" -) - type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { BaseImporter exchangers.BaseRequestExchanger - - registerSiteActionHandler queryCallback - unregisterSiteActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { body, _ := ioutil.ReadAll(req.Body) - meshData, status, respData, err := importer.handleQuery(body, req.URL.Path, req.URL.Query()) + meshDataSet, status, respData, err := importer.handleQuery(body, req.URL.Query()) if err == nil { - if len(meshData) > 0 { - importer.mergeImportedMeshData(meshData) + if len(meshDataSet) > 0 { + importer.mergeImportedMeshDataSet(meshDataSet) } } else { respData = []byte(err.Error()) @@ -60,35 +50,19 @@ func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req _, _ = resp.Write(respData) } -func (importer *BaseRequestImporter) mergeImportedMeshData(meshData meshdata.Vector) { +func (importer *BaseRequestImporter) mergeImportedMeshDataSet(meshDataSet meshdata.Vector) { // Merge the newly imported data with any existing data stored in the importer if importer.meshData != nil { // Need to manually lock the data for writing importer.Locker().Lock() defer importer.Locker().Unlock() - importer.meshData = append(importer.meshData, meshData...) + importer.meshData = append(importer.meshData, meshDataSet...) } else { - importer.SetMeshData(meshData) // SetMeshData will do the locking itself + importer.SetMeshData(meshDataSet) // SetMeshData will do the locking itself } } -func (importer *BaseRequestImporter) handleQuery(data []byte, path string, params url.Values) (meshdata.Vector, int, []byte, error) { - action := params.Get("action") - switch strings.ToLower(action) { - case queryActionRegisterSite: - if importer.registerSiteActionHandler != nil { - return importer.registerSiteActionHandler(data, params) - } - - case queryActionUnregisterSite: - if importer.unregisterSiteActionHandler != nil { - return importer.unregisterSiteActionHandler(data, params) - } - - default: - return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) - } - - return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", action) +func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + return importer.HandleAction(nil, data, params) } diff --git a/pkg/mentix/exchangers/importers/webapi.go b/pkg/mentix/exchangers/importers/webapi.go index 508ff62643..2a326d8786 100755 --- a/pkg/mentix/exchangers/importers/webapi.go +++ b/pkg/mentix/exchangers/importers/webapi.go @@ -40,8 +40,8 @@ func (importer *WebAPIImporter) Activate(conf *config.Configuration, log *zerolo importer.SetEndpoint(conf.Importers.WebAPI.Endpoint, conf.Importers.WebAPI.IsProtected) importer.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) - importer.registerSiteActionHandler = webapi.HandleRegisterSiteQuery - importer.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery + importer.RegisterActionHandler("register", webapi.HandleRegisterSiteQuery) + importer.RegisterActionHandler("unregister", webapi.HandleUnregisterSiteQuery) return nil } diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index 868d0c8cc3..6b21f2d0b9 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -59,11 +59,11 @@ func handleQuery(data []byte, params url.Values, status int, msg string) (meshda } // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/exchangers/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go index c4c9e705af..d5e21eb75e 100644 --- a/pkg/mentix/exchangers/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -19,8 +19,12 @@ package exchangers import ( + "fmt" "net/http" + "net/url" "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" ) // RequestExchanger is the interface implemented by exchangers that offer an HTTP endpoint. @@ -35,12 +39,16 @@ type RequestExchanger interface { HandleRequest(resp http.ResponseWriter, req *http.Request) } +type queryCallback func(*meshdata.MeshData, []byte, url.Values) (meshdata.Vector, int, []byte, error) + // BaseRequestExchanger implements basic exporter functionality common to all request exporters. type BaseRequestExchanger struct { RequestExchanger endpoint string isProtectedEndpoint bool + + actionHandlers map[string]queryCallback } // Endpoint returns the (relative) endpoint of the exchanger. @@ -73,3 +81,23 @@ func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool { func (exchanger *BaseRequestExchanger) HandleRequest(resp http.ResponseWriter, req *http.Request) error { return nil } + +// RegisterActionHandler registers a new handler for the specified action. +func (exchanger *BaseRequestExchanger) RegisterActionHandler(action string, callback queryCallback) { + if exchanger.actionHandlers == nil { + exchanger.actionHandlers = make(map[string]queryCallback) + } + exchanger.actionHandlers[action] = callback +} + +// HandleAction executes the registered handler for the specified action, if any. +func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + reqAction := params.Get("action") + for action, handler := range exchanger.actionHandlers { + if strings.EqualFold(action, reqAction) { + return handler(meshData, body, params) + } + } + + return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", reqAction) +} From dc5f93cf5bc10dae691f0d971a521b966fb9eead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 15:35:06 +0100 Subject: [PATCH 37/59] Add site registration importer --- internal/http/services/mentix/mentix.go | 4 + pkg/mentix/config/config.go | 6 + pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/gocdb.go | 2 +- pkg/mentix/connectors/gocdb/query.go | 2 +- .../exchangers/importers/reqimporter.go | 2 - pkg/mentix/exchangers/importers/sitereg.go | 61 ++++ .../exchangers/importers/sitereg/query.go | 121 +++++++ .../exchangers/importers/sitereg/types.go | 143 ++++++++ .../exchangers/importers/webapi/query.go | 6 +- pkg/mentix/meshdata/service.go | 2 +- pkg/mentix/meshdata/site.go | 4 +- pkg/mentix/utils/countries/countries.go | 316 ++++++++++++++++++ .../utils.go => utils/network/network.go} | 4 + 14 files changed, 665 insertions(+), 10 deletions(-) create mode 100644 pkg/mentix/exchangers/importers/sitereg.go create mode 100755 pkg/mentix/exchangers/importers/sitereg/query.go create mode 100644 pkg/mentix/exchangers/importers/sitereg/types.go create mode 100644 pkg/mentix/utils/countries/countries.go rename pkg/mentix/{network/utils.go => utils/network/network.go} (97%) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index b33a934348..b33548ae54 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -141,6 +141,10 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Importers.WebAPI.Endpoint = "/sites" } + if conf.Importers.SiteRegistration.Endpoint == "" { + conf.Importers.SiteRegistration.Endpoint = "/sitereg" + } + // Exporters addDefaultConnector := func(enabledList *[]string) { if len(*enabledList) == 0 { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 806c1015f9..b099f1cc5e 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -41,6 +41,12 @@ type Configuration struct { IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` + + SiteRegistration struct { + Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` + } `mapstructure:"sitereg"` } `mapstructure:"importers"` Exporters struct { diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 04ff545ec3..4f6dea0883 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,6 +28,8 @@ const ( const ( // ImporterIDWebAPI is the identifier for the WebAPI importer. ImporterIDWebAPI = "webapi" + // ImporterIDSiteRegistration is the identifier for the external site registration importer. + ImporterIDSiteRegistration = "sitereg" ) const ( diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 5d5c50b3a7..1ac667dc9d 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors/gocdb" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/network" + "github.com/cs3org/reva/pkg/mentix/utils/network" ) // GOCDBConnector is used to read mesh data from a GOCDB instance. diff --git a/pkg/mentix/connectors/gocdb/query.go b/pkg/mentix/connectors/gocdb/query.go index 8a188489b4..9ee0c420ab 100755 --- a/pkg/mentix/connectors/gocdb/query.go +++ b/pkg/mentix/connectors/gocdb/query.go @@ -21,7 +21,7 @@ package gocdb import ( "fmt" - "github.com/cs3org/reva/pkg/mentix/network" + "github.com/cs3org/reva/pkg/mentix/utils/network" ) // QueryGOCDB retrieves data from one of GOCDB's endpoints. diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 1604c857f1..d229de834d 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -27,8 +27,6 @@ import ( "github.com/cs3org/reva/pkg/mentix/meshdata" ) -type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) - // BaseRequestImporter implements basic importer functionality common to all request importers. type BaseRequestImporter struct { BaseImporter diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go new file mode 100644 index 0000000000..1918147bdb --- /dev/null +++ b/pkg/mentix/exchangers/importers/sitereg.go @@ -0,0 +1,61 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchangers/importers/sitereg" +) + +// SiteRegistrationImporter implements the external site registration importer. +type SiteRegistrationImporter struct { + BaseRequestImporter +} + +// Activate activates the importer. +func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { + return err + } + + // Store SiteRegistration specifics + importer.SetEndpoint(conf.Importers.SiteRegistration.Endpoint, conf.Importers.SiteRegistration.IsProtected) + importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) + + importer.RegisterActionHandler("register", sitereg.HandleRegisterSiteQuery) + importer.RegisterActionHandler("unregister", sitereg.HandleUnregisterSiteQuery) + + return nil +} + +// GetID returns the ID of the importer. +func (importer *SiteRegistrationImporter) GetID() string { + return config.ImporterIDSiteRegistration +} + +// GetName returns the display name of the importer. +func (importer *SiteRegistrationImporter) GetName() string { + return "SiteRegistration" +} + +func init() { + registerImporter(&SiteRegistrationImporter{}) +} diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go new file mode 100755 index 0000000000..4f1a8c8790 --- /dev/null +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -0,0 +1,121 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sitereg + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/mentix/key" + "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/utils/network" +) + +func decodeQueryData(data []byte) (*siteRegistrationData, error) { + siteData := &siteRegistrationData{} + if err := json.Unmarshal(data, siteData); err != nil { + return nil, err + } + + if err := siteData.Verify(); err != nil { + return nil, errors.Wrap(err, "verifying the imported site data failed") + } + + return siteData, nil +} + +func decodeAPIKey(params url.Values) (key.SiteIdentifier, int8, error) { + apiKey := params.Get("apiKey") + if len(apiKey) == 0 { + return "", 0, errors.Errorf("no API key specified") + } + + // TODO: Check & verify API key (does exist?), query user data etc. blabla (depending on key flags) + + _, flags, _, err := key.SplitAPIKey(apiKey) + if err != nil { + return "", 0, errors.Errorf("sticky API key specified") + } + + siteID, err := key.CalculateSiteID(apiKey) + if err != nil { + return "", 0, errors.Wrap(err, "unable to get site ID") + } + + return siteID, flags, nil +} + +func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, error) { + return nil, http.StatusBadRequest, network.CreateResponse(msg, network.ResponseParams{"error": err.Error()}), nil +} + +// HandleRegisterSiteQuery registers a site. +func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + siteID, flags, err := decodeAPIKey(params) + if err != nil { + return createErrorResponse("INVALID_API_KEY", err) + } + + // TODO: Check if site with ID already exists; bail out if so (or update, whatever) + + // Decode the site registration data and convert it to a meshdata object + siteData, err := decodeQueryData(data) + if err != nil { + return createErrorResponse("INVALID_SITE_DATA", err) + } + + siteType := meshdata.SiteTypeCommunity + if flags&key.FlagScienceMesh == key.FlagScienceMesh { + siteType = meshdata.SiteTypeScienceMesh + } + + site, err := siteData.ToMeshDataSite(siteID, siteType) + if err != nil { + return createErrorResponse("INVALID_SITE_DATA", err) + } + + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + if err := meshData.Verify(); err != nil { + return createErrorResponse("INVALID_MESH_DATA", err) + } + meshData.Status = meshdata.StatusDefault + meshData.InferMissingData() + + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse("SITE_REGISTERED", network.ResponseParams{"id": siteID}), nil +} + +// HandleUnregisterSiteQuery unregisters a site. +func HandleUnregisterSiteQuery(_ *meshdata.MeshData, _ []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + siteID, _, err := decodeAPIKey(params) + if err != nil { + return createErrorResponse("INVALID_API_KEY", err) + } + + // TODO: Check if site with ID exists; bail out if not + + // To remove a site, a meshdata object that contains a site with the given ID needs to be created + site := &meshdata.Site{ID: siteID} + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + meshData.Status = meshdata.StatusObsolete + + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse("SITE_UNREGISTERED", network.ResponseParams{"id": siteID}), nil +} diff --git a/pkg/mentix/exchangers/importers/sitereg/types.go b/pkg/mentix/exchangers/importers/sitereg/types.go new file mode 100644 index 0000000000..8061ba6f13 --- /dev/null +++ b/pkg/mentix/exchangers/importers/sitereg/types.go @@ -0,0 +1,143 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sitereg + +import ( + "net/url" + + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/mentix/key" + "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/utils/countries" + "github.com/cs3org/reva/pkg/mentix/utils/network" +) + +type siteRegistrationData struct { + Name string `json:"name"` + URL string `json:"url"` + CountryCode string `json:"countryCode"` + + Reva struct { + Host string `json:"host"` + URL string `json:"url"` + MetricsPath string `json:"metricsPath"` + } `json:"reva"` +} + +/* Example JSON: +{ + "name": "Testsite", + "url": "https://test-site.de/owncloud", + "countryCode": "DE", + "reva": { + "url": "https://test-site.de/owncloud/reva", + "metricsPath": "/iop/metrics" + } +} +*/ + +// Verify checks whether the entered data is valid and complete. +func (siteData *siteRegistrationData) Verify() error { + if len(siteData.Name) == 0 { + return errors.Errorf("no site name provided") + } + if len(siteData.URL) > 0 { + if _, err := url.Parse(siteData.URL); err != nil { + return errors.Wrap(err, "invalid site URL provided") + } + } else { + return errors.Errorf("no site URL provided") + } + + if len(siteData.Reva.Host) == 0 && len(siteData.Reva.URL) == 0 { + return errors.Errorf("no Reva host or URL provided") + } + if len(siteData.Reva.URL) > 0 { + if _, err := url.Parse(siteData.Reva.URL); err != nil { + return errors.Wrap(err, "invalid Reva URL provided") + } + } + if len(siteData.Reva.MetricsPath) == 0 { + return errors.Errorf("no Reva metrics path provided") + } + + return nil +} + +// ToMeshDataSite converts the stored data into a meshdata site object, filling out as much data as possible. +func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, siteType meshdata.SiteType) (*meshdata.Site, error) { + siteUrl, err := url.Parse(siteData.URL) + if err != nil { + return nil, errors.Wrap(err, "invalid site URL") + } + + // Create the Reva service entry + revaHost := siteData.Reva.Host + revaUrl := siteData.Reva.URL + + if len(revaHost) == 0 { // Infer host from URL + URL, _ := url.Parse(revaUrl) + revaHost = network.ExtractDomainFromURL(URL, true) + } else if len(revaUrl) == 0 { // Infer URL from host + URL, _ := network.GenerateURL(revaHost, "", network.URLParams{}) + revaUrl = URL.String() + } + + properties := make(map[string]string, 1) + meshdata.SetPropertyValue(&properties, meshdata.PropertyMetricsPath, siteData.Reva.MetricsPath) + + revaService := &meshdata.Service{ + ServiceEndpoint: &meshdata.ServiceEndpoint{ + Type: &meshdata.ServiceType{ + Name: "REVAD", + Description: "Reva Daemon", + }, + Name: revaHost + " - REVAD", + URL: revaUrl, + IsMonitored: true, + Properties: properties, + }, + Host: revaHost, + AdditionalEndpoints: nil, + } + + // Create the site data + site := &meshdata.Site{ + Type: siteType, + ID: siteID, + Name: siteData.Name, + FullName: siteData.Name, + Organization: "", + Domain: network.ExtractDomainFromURL(siteUrl, true), + Homepage: siteData.URL, + Email: "", // TODO: Get account data + Description: siteData.Name + " @ " + siteData.URL, + Country: countries.LookupCountry(siteData.CountryCode), + CountryCode: siteData.CountryCode, + Location: "", + Latitude: 0, + Longitude: 0, + Services: []*meshdata.Service{revaService}, + Properties: nil, + } + + site.InferMissingData() + return site, nil +} diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index 6b21f2d0b9..19e1dca988 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -25,7 +25,7 @@ import ( "net/url" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/network" + "github.com/cs3org/reva/pkg/mentix/utils/network" ) func decodeQueryData(data []byte) (*meshdata.MeshData, error) { @@ -59,11 +59,11 @@ func handleQuery(data []byte, params url.Values, status int, msg string) (meshda } // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleUnregisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 30af06f408..c56428500b 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -22,7 +22,7 @@ import ( "fmt" "net/url" - "github.com/cs3org/reva/pkg/mentix/network" + "github.com/cs3org/reva/pkg/mentix/utils/network" ) // Service represents a service managed by Mentix. diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index b3524795c4..35d9cc0248 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -23,12 +23,12 @@ import ( "net/url" "strings" - "github.com/cs3org/reva/pkg/mentix/network" + "github.com/cs3org/reva/pkg/mentix/utils/network" ) const ( // SiteTypeScienceMesh flags a site as being part of the mesh. - SiteTypeScienceMesh = iota + SiteTypeScienceMesh SiteType = iota // SiteTypeCommunity flags a site as being a community site. SiteTypeCommunity ) diff --git a/pkg/mentix/utils/countries/countries.go b/pkg/mentix/utils/countries/countries.go new file mode 100644 index 0000000000..0fd7664ce6 --- /dev/null +++ b/pkg/mentix/utils/countries/countries.go @@ -0,0 +1,316 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package countries + +import ( + "strconv" + "strings" +) + +const countriesData = ` +Afghanistan AF AFG 004 +Albania AL ALB 008 +Algeria DZ DZA 012 +American Samoa AS ASM 016 +Andorra AD AND 020 +Angola AO AGO 024 +Anguilla AI AIA 660 +Antarctica AQ ATA 010 +Antigua and Barbuda AG ATG 028 +Argentina AR ARG 032 +Armenia AM ARM 051 +Aruba AW ABW 533 +Australia AU AUS 036 +Austria AT AUT 040 +Azerbaijan AZ AZE 031 +Bahamas (the) BS BHS 044 +Bahrain BH BHR 048 +Bangladesh BD BGD 050 +Barbados BB BRB 052 +Belarus BY BLR 112 +Belgium BE BEL 056 +Belize BZ BLZ 084 +Benin BJ BEN 204 +Bermuda BM BMU 060 +Bhutan BT BTN 064 +Bolivia (Plurinational State of) BO BOL 068 +Bonaire, Sint Eustatius and Saba BQ BES 535 +Bosnia and Herzegovina BA BIH 070 +Botswana BW BWA 072 +Bouvet Island BV BVT 074 +Brazil BR BRA 076 +British Indian Ocean Territory (the) IO IOT 086 +Brunei Darussalam BN BRN 096 +Bulgaria BG BGR 100 +Burkina Faso BF BFA 854 +Burundi BI BDI 108 +Cabo Verde CV CPV 132 +Cambodia KH KHM 116 +Cameroon CM CMR 120 +Canada CA CAN 124 +Cayman Islands (the) KY CYM 136 +Central African Republic (the) CF CAF 140 +Chad TD TCD 148 +Chile CL CHL 152 +China CN CHN 156 +Christmas Island CX CXR 162 +Cocos (Keeling) Islands (the) CC CCK 166 +Colombia CO COL 170 +Comoros (the) KM COM 174 +Congo (the Democratic Republic of the) CD COD 180 +Congo (the) CG COG 178 +Cook Islands (the) CK COK 184 +Costa Rica CR CRI 188 +Croatia HR HRV 191 +Cuba CU CUB 192 +Curaçao CW CUW 531 +Cyprus CY CYP 196 +Czechia CZ CZE 203 +Côte d'Ivoire CI CIV 384 +Denmark DK DNK 208 +Djibouti DJ DJI 262 +Dominica DM DMA 212 +Dominican Republic (the) DO DOM 214 +Ecuador EC ECU 218 +Egypt EG EGY 818 +El Salvador SV SLV 222 +Equatorial Guinea GQ GNQ 226 +Eritrea ER ERI 232 +Estonia EE EST 233 +Eswatini SZ SWZ 748 +Ethiopia ET ETH 231 +Falkland Islands (the) [Malvinas] FK FLK 238 +Faroe Islands (the) FO FRO 234 +Fiji FJ FJI 242 +Finland FI FIN 246 +France FR FRA 250 +French Guiana GF GUF 254 +French Polynesia PF PYF 258 +French Southern Territories (the) TF ATF 260 +Gabon GA GAB 266 +Gambia (the) GM GMB 270 +Georgia GE GEO 268 +Germany DE DEU 276 +Ghana GH GHA 288 +Gibraltar GI GIB 292 +Greece GR GRC 300 +Greenland GL GRL 304 +Grenada GD GRD 308 +Guadeloupe GP GLP 312 +Guam GU GUM 316 +Guatemala GT GTM 320 +Guernsey GG GGY 831 +Guinea GN GIN 324 +Guinea-Bissau GW GNB 624 +Guyana GY GUY 328 +Haiti HT HTI 332 +Heard Island and McDonald Islands HM HMD 334 +Holy See (the) VA VAT 336 +Honduras HN HND 340 +Hong Kong HK HKG 344 +Hungary HU HUN 348 +Iceland IS ISL 352 +India IN IND 356 +Indonesia ID IDN 360 +Iran (Islamic Republic of) IR IRN 364 +Iraq IQ IRQ 368 +Ireland IE IRL 372 +Isle of Man IM IMN 833 +Israel IL ISR 376 +Italy IT ITA 380 +Jamaica JM JAM 388 +Japan JP JPN 392 +Jersey JE JEY 832 +Jordan JO JOR 400 +Kazakhstan KZ KAZ 398 +Kenya KE KEN 404 +Kiribati KI KIR 296 +Korea (the Democratic People's Republic of) KP PRK 408 +Korea (the Republic of) KR KOR 410 +Kuwait KW KWT 414 +Kyrgyzstan KG KGZ 417 +Lao People's Democratic Republic (the) LA LAO 418 +Latvia LV LVA 428 +Lebanon LB LBN 422 +Lesotho LS LSO 426 +Liberia LR LBR 430 +Libya LY LBY 434 +Liechtenstein LI LIE 438 +Lithuania LT LTU 440 +Luxembourg LU LUX 442 +Macao MO MAC 446 +Madagascar MG MDG 450 +Malawi MW MWI 454 +Malaysia MY MYS 458 +Maldives MV MDV 462 +Mali ML MLI 466 +Malta MT MLT 470 +Marshall Islands (the) MH MHL 584 +Martinique MQ MTQ 474 +Mauritania MR MRT 478 +Mauritius MU MUS 480 +Mayotte YT MYT 175 +Mexico MX MEX 484 +Micronesia (Federated States of) FM FSM 583 +Moldova (the Republic of) MD MDA 498 +Monaco MC MCO 492 +Mongolia MN MNG 496 +Montenegro ME MNE 499 +Montserrat MS MSR 500 +Morocco MA MAR 504 +Mozambique MZ MOZ 508 +Myanmar MM MMR 104 +Namibia NA NAM 516 +Nauru NR NRU 520 +Nepal NP NPL 524 +Netherlands (the) NL NLD 528 +New Caledonia NC NCL 540 +New Zealand NZ NZL 554 +Nicaragua NI NIC 558 +Niger (the) NE NER 562 +Nigeria NG NGA 566 +Niue NU NIU 570 +Norfolk Island NF NFK 574 +Northern Mariana Islands (the) MP MNP 580 +Norway NO NOR 578 +Oman OM OMN 512 +Pakistan PK PAK 586 +Palau PW PLW 585 +Palestine, State of PS PSE 275 +Panama PA PAN 591 +Papua New Guinea PG PNG 598 +Paraguay PY PRY 600 +Peru PE PER 604 +Philippines (the) PH PHL 608 +Pitcairn PN PCN 612 +Poland PL POL 616 +Portugal PT PRT 620 +Puerto Rico PR PRI 630 +Qatar QA QAT 634 +Republic of North Macedonia MK MKD 807 +Romania RO ROU 642 +Russian Federation (the) RU RUS 643 +Rwanda RW RWA 646 +Réunion RE REU 638 +Saint Barthélemy BL BLM 652 +Saint Helena, Ascension and Tristan da Cunha SH SHN 654 +Saint Kitts and Nevis KN KNA 659 +Saint Lucia LC LCA 662 +Saint Martin (French part) MF MAF 663 +Saint Pierre and Miquelon PM SPM 666 +Saint Vincent and the Grenadines VC VCT 670 +Samoa WS WSM 882 +San Marino SM SMR 674 +Sao Tome and Principe ST STP 678 +Saudi Arabia SA SAU 682 +Senegal SN SEN 686 +Serbia RS SRB 688 +Seychelles SC SYC 690 +Sierra Leone SL SLE 694 +Singapore SG SGP 702 +Sint Maarten (Dutch part) SX SXM 534 +Slovakia SK SVK 703 +Slovenia SI SVN 705 +Solomon Islands SB SLB 090 +Somalia SO SOM 706 +South Africa ZA ZAF 710 +South Georgia and the South Sandwich Islands GS SGS 239 +South Sudan SS SSD 728 +Spain ES ESP 724 +Sri Lanka LK LKA 144 +Sudan (the) SD SDN 729 +Suriname SR SUR 740 +Svalbard and Jan Mayen SJ SJM 744 +Sweden SE SWE 752 +Switzerland CH CHE 756 +Syrian Arab Republic SY SYR 760 +Taiwan (Province of China) TW TWN 158 +Tajikistan TJ TJK 762 +Tanzania, United Republic of TZ TZA 834 +Thailand TH THA 764 +Timor-Leste TL TLS 626 +Togo TG TGO 768 +Tokelau TK TKL 772 +Tonga TO TON 776 +Trinidad and Tobago TT TTO 780 +Tunisia TN TUN 788 +Turkey TR TUR 792 +Turkmenistan TM TKM 795 +Turks and Caicos Islands (the) TC TCA 796 +Tuvalu TV TUV 798 +Uganda UG UGA 800 +Ukraine UA UKR 804 +United Arab Emirates (the) AE ARE 784 +United Kingdom of Great Britain and Northern Ireland (the) GB GBR 826 +United States Minor Outlying Islands (the) UM UMI 581 +United States of America (the) US USA 840 +Uruguay UY URY 858 +Uzbekistan UZ UZB 860 +Vanuatu VU VUT 548 +Venezuela (Bolivarian Republic of) VE VEN 862 +Viet Nam VN VNM 704 +Virgin Islands (British) VG VGB 092 +Virgin Islands (U.S.) VI VIR 850 +Wallis and Futuna WF WLF 876 +Western Sahara EH ESH 732 +Yemen YE YEM 887 +Zambia ZM ZMB 894 +Zimbabwe ZW ZWE 716 +Åland Islands AX ALA 248 +` + +type countryCode struct { + Alpha2 string + Alpha3 string + Numerical int +} + +var countryCodeTable map[string]countryCode + +// LookupCountry searches for the country specified by the given code (Alpha2/3 or numerical). +// If no country with the code exists, an empty string is returned. +func LookupCountry(code string) string { + numerical, err := strconv.Atoi(code) + if err != nil { + numerical = -1 + } + + for name, cc := range countryCodeTable { + if strings.EqualFold(code, cc.Alpha2) || strings.EqualFold(code, cc.Alpha3) || cc.Numerical == numerical { + return name + } + } + return "" +} + +func init() { + countryCodeTable = make(map[string]countryCode, 250) + for _, countryData := range strings.Split(countriesData, "\n") { + tokens := strings.Split(countryData, "\t") + if len(tokens) == 4 { + numerical, _ := strconv.Atoi(tokens[3]) + + countryCodeTable[tokens[0]] = countryCode{ + Alpha2: tokens[1], + Alpha3: tokens[2], + Numerical: numerical, + } + } + } +} diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/utils/network/network.go similarity index 97% rename from pkg/mentix/network/utils.go rename to pkg/mentix/utils/network/network.go index c2c9ac7b2c..b8e0cbb81d 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/utils/network/network.go @@ -42,6 +42,10 @@ func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", host, path, params) } + if len(fullURL.Scheme) == 0 { + fullURL.Scheme = "https" + } + fullURL.Path = p.Join(fullURL.Path, path) query := make(url.Values) From 4e4211c2042f4f4a7e46c81005da81166286bcc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 23 Feb 2021 17:24:15 +0100 Subject: [PATCH 38/59] Enhance site ID security --- internal/http/services/accounts/data/account.go | 3 ++- pkg/mentix/exchangers/importers/sitereg/query.go | 11 ++++++++++- pkg/mentix/key/siteid.go | 5 +++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/http/services/accounts/data/account.go b/internal/http/services/accounts/data/account.go index 47ff80a0ea..c3f11f9435 100644 --- a/internal/http/services/accounts/data/account.go +++ b/internal/http/services/accounts/data/account.go @@ -19,6 +19,7 @@ package data import ( + "strings" "time" "github.com/pkg/errors" @@ -49,7 +50,7 @@ type AccountData struct { type Accounts = []*Account func (acc *Account) GetSiteID() key.SiteIdentifier { - if id, err := key.CalculateSiteID(acc.Data.APIKey); err == nil { + if id, err := key.CalculateSiteID(acc.Data.APIKey, strings.ToLower(acc.Email)); err == nil { return id } diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index 4f1a8c8790..bca5f8c6cb 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -22,6 +22,7 @@ import ( "encoding/json" "net/http" "net/url" + "strings" "github.com/pkg/errors" @@ -56,7 +57,10 @@ func decodeAPIKey(params url.Values) (key.SiteIdentifier, int8, error) { return "", 0, errors.Errorf("sticky API key specified") } - siteID, err := key.CalculateSiteID(apiKey) + // TODO: Extract email from account + email := "" + + siteID, err := key.CalculateSiteID(apiKey, strings.ToLower(email)) if err != nil { return "", 0, errors.Wrap(err, "unable to get site ID") } @@ -112,6 +116,11 @@ func HandleUnregisterSiteQuery(_ *meshdata.MeshData, _ []byte, params url.Values // TODO: Check if site with ID exists; bail out if not + // The site ID must be provided in the call as well to enhance security further + if params.Get("siteId") != siteID { + return createErrorResponse("INVALID_SITE_ID", errors.Errorf("site ID mismatch")) + } + // To remove a site, a meshdata object that contains a site with the given ID needs to be created site := &meshdata.Site{ID: siteID} meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} diff --git a/pkg/mentix/key/siteid.go b/pkg/mentix/key/siteid.go index bc8af06c98..fda47f4426 100644 --- a/pkg/mentix/key/siteid.go +++ b/pkg/mentix/key/siteid.go @@ -29,14 +29,15 @@ import ( type SiteIdentifier = string // CalculateSiteID calculates a (stable) site ID from the given API key. -// The site ID is actually the CRC64 hash of the provided API key, thus it is stable for any given key. -func CalculateSiteID(apiKey APIKey) (SiteIdentifier, error) { +// The site ID is actually the CRC64 hash of the provided API key plus a salt value, thus it is stable for any given key & salt pair. +func CalculateSiteID(apiKey APIKey, salt string) (SiteIdentifier, error) { if len(apiKey) != apiKeyLength { return "", errors.Errorf("invalid API key length") } hash := crc64.New(crc64.MakeTable(crc64.ECMA)) _, _ = hash.Write([]byte(apiKey)) + _, _ = hash.Write([]byte(salt)) value := hash.Sum(nil) return fmt.Sprintf("%4x-%4x-%4x-%4x", value[:2], value[2:4], value[4:6], value[6:]), nil } From e695a28d266c5ca7c4ff071d7726b0a3e2eb0e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 10:56:14 +0100 Subject: [PATCH 39/59] Use accounts service for API key querying --- pkg/mentix/config/config.go | 6 ++ pkg/mentix/connectors/gocdb/query.go | 7 +- pkg/mentix/exchangers/importers/sitereg.go | 5 + .../importers/sitereg/accservice.go | 99 +++++++++++++++++++ .../exchangers/importers/sitereg/query.go | 21 +++- pkg/mentix/utils/network/network.go | 20 +++- 6 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 pkg/mentix/exchangers/importers/sitereg/accservice.go diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index b099f1cc5e..639140394e 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -75,6 +75,12 @@ type Configuration struct { } `mapstructure:"promsd"` } `mapstructure:"exporters"` + AccountsService struct { + URL string `mapstructure:"url"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + } `mapstructure:"accounts"` + // Internal settings EnabledConnectors []string `mapstructure:"-"` EnabledImporters []string `mapstructure:"-"` diff --git a/pkg/mentix/connectors/gocdb/query.go b/pkg/mentix/connectors/gocdb/query.go index 9ee0c420ab..a586e38cb7 100755 --- a/pkg/mentix/connectors/gocdb/query.go +++ b/pkg/mentix/connectors/gocdb/query.go @@ -43,7 +43,12 @@ func QueryGOCDB(address string, method string, isPrivate bool, scope string, par } // Query the data from GOCDB - data, err := network.ReadEndpoint(address, path, params) + endpointURL, err := network.GenerateURL(address, path, params) + if err != nil { + return nil, fmt.Errorf("unable to generate the GOCDB URL: %v", err) + } + + data, err := network.ReadEndpoint(endpointURL, nil, true) if err != nil { return nil, fmt.Errorf("unable to read GOCDB endpoint: %v", err) } diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go index 1918147bdb..90de14195e 100644 --- a/pkg/mentix/exchangers/importers/sitereg.go +++ b/pkg/mentix/exchangers/importers/sitereg.go @@ -19,6 +19,7 @@ package importers import ( + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -36,6 +37,10 @@ func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, l return err } + if err := sitereg.ConfigureAccountsService(conf.AccountsService.URL, conf.AccountsService.User, conf.AccountsService.Password); err != nil { + return errors.Wrap(err, "error while configuring the accounts service") + } + // Store SiteRegistration specifics importer.SetEndpoint(conf.Importers.SiteRegistration.Endpoint, conf.Importers.SiteRegistration.IsProtected) importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) diff --git a/pkg/mentix/exchangers/importers/sitereg/accservice.go b/pkg/mentix/exchangers/importers/sitereg/accservice.go new file mode 100644 index 0000000000..7f670d9725 --- /dev/null +++ b/pkg/mentix/exchangers/importers/sitereg/accservice.go @@ -0,0 +1,99 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sitereg + +import ( + "encoding/json" + "fmt" + "net/url" + "path" + "strings" + + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/mentix/utils/network" +) + +type accountsServiceSettings struct { + URL *url.URL + user string + password string +} + +type requestResponse struct { + Success bool + Error string + Data interface{} +} + +var ( + accountsService accountsServiceSettings +) + +// ConfigureAccountsService configures the accounts service for the site registration importer. +func ConfigureAccountsService(address string, user string, password string) error { + endpointURL, err := url.Parse(address) + if err != nil { + return errors.Wrap(err, "unable to parse the accounts service URL") + } + + accountsService.URL = endpointURL + accountsService.user = user + accountsService.password = password + + return nil +} + +func queryAccountsService(endpoint string, params network.URLParams) (*requestResponse, error) { + fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", accountsService.URL.Scheme, accountsService.URL.Host), path.Join(accountsService.URL.Path, endpoint), params) + if err != nil { + return nil, errors.Wrap(err, "error while building the service accounts query URL") + } + + data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: accountsService.user, Password: accountsService.password}, false) + if err != nil { + return nil, errors.Wrap(err, "unable to query the service accounts endpoint") + } + + resp := &requestResponse{} + if err := json.Unmarshal(data, resp); err != nil { + return nil, errors.Wrap(err, "unable to unmarshal response data") + } + return resp, nil +} + +func getResponseValue(resp *requestResponse, path string) interface{} { + if data, ok := resp.Data.(map[string]interface{}); ok { + tokens := strings.Split(path, ".") + for i, name := range tokens { + if i == len(tokens)-1 { + if value, ok := data[name]; ok { + return value + } + } + + if data, ok = data[name].(map[string]interface{}); !ok { + break + } + + } + } + + return nil +} diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index bca5f8c6cb..7dac4302e4 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -50,16 +50,29 @@ func decodeAPIKey(params url.Values) (key.SiteIdentifier, int8, error) { return "", 0, errors.Errorf("no API key specified") } - // TODO: Check & verify API key (does exist?), query user data etc. blabla (depending on key flags) + // Try to get an account that is associated with the given API key; if none exists, return an error + resp, err := queryAccountsService("find", network.URLParams{"by": "apikey", "value": apiKey}) + if err != nil { + return "", 0, errors.Wrap(err, "error while querying the accounts service") + } + if !resp.Success { + return "", 0, errors.Errorf("unable to fetch account associated with the provided API key: %v", resp.Error) + } + + // Extract email from account data; this is needed to calculate the site ID from the API key + email := "" + if value := getResponseValue(resp, "account.email"); value != nil { + email, _ = value.(string) + } + if len(email) == 0 { + return "", 0, errors.Errorf("could not get the email address of the user account") + } _, flags, _, err := key.SplitAPIKey(apiKey) if err != nil { return "", 0, errors.Errorf("sticky API key specified") } - // TODO: Extract email from account - email := "" - siteID, err := key.CalculateSiteID(apiKey, strings.ToLower(email)) if err != nil { return "", 0, errors.Wrap(err, "unable to get site ID") diff --git a/pkg/mentix/utils/network/network.go b/pkg/mentix/utils/network/network.go index b8e0cbb81d..9704b8b578 100644 --- a/pkg/mentix/utils/network/network.go +++ b/pkg/mentix/utils/network/network.go @@ -35,6 +35,11 @@ type URLParams map[string]string // ResponseParams holds parameters of an HTTP response. type ResponseParams map[string]interface{} +type BasicAuth struct { + User string + Password string +} + // GenerateURL creates a URL object from a host, path and optional parameters. func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { fullURL, err := url.Parse(host) @@ -58,18 +63,23 @@ func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { } // ReadEndpoint reads data from an HTTP endpoint. -func ReadEndpoint(host string, path string, params URLParams) ([]byte, error) { - endpointURL, err := GenerateURL(host, path, params) +func ReadEndpoint(endpointURL *url.URL, auth *BasicAuth, checkStatus bool) ([]byte, error) { + // Prepare the request + req, err := http.NewRequest("GET", endpointURL.String(), nil) if err != nil { - return nil, fmt.Errorf("unable to generate endpoint URL: %v", err) + return nil, fmt.Errorf("unable to create HTTP request: %v", err) + } + + if auth != nil { + req.SetBasicAuth(auth.User, auth.Password) } // Fetch the data and read the body - resp, err := http.Get(endpointURL.String()) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("unable to get data from endpoint: %v", err) } - if resp.StatusCode >= 400 { + if checkStatus && resp.StatusCode >= 400 { return nil, fmt.Errorf("invalid response received: %v", resp.Status) } From fcecdef3fca2dfe7dd0e65eb872afdaafb60944e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 12:21:40 +0100 Subject: [PATCH 40/59] All exchangers now store the current mesh data --- pkg/mentix/exchangers/exchanger.go | 72 +++++++++++++++++++ pkg/mentix/exchangers/exporters/exporter.go | 72 ------------------- pkg/mentix/exchangers/exporters/webapi.go | 3 +- pkg/mentix/exchangers/importers/importer.go | 36 +++++----- .../exchangers/importers/reqimporter.go | 10 +-- pkg/mentix/exchangers/importers/sitereg.go | 1 + .../exchangers/importers/sitereg/query.go | 2 - pkg/mentix/mentix.go | 10 ++- 8 files changed, 105 insertions(+), 101 deletions(-) diff --git a/pkg/mentix/exchangers/exchanger.go b/pkg/mentix/exchangers/exchanger.go index e3a3f2f534..a017f501cb 100644 --- a/pkg/mentix/exchangers/exchanger.go +++ b/pkg/mentix/exchangers/exchanger.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/entity" + "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Exchanger is the base interface for importers and exporters. @@ -37,6 +38,12 @@ type Exchanger interface { Start() error // Stop stops any running background activities of the exchanger. Stop() + + // MeshData returns the mesh data. + MeshData() *meshdata.MeshData + + // Update is called whenever the mesh data set has changed to reflect these changes. + Update(meshdata.Map) error } // BaseExchanger implements basic exchanger functionality common to all exchangers. @@ -48,6 +55,9 @@ type BaseExchanger struct { enabledConnectors []string + meshData *meshdata.MeshData + allowUnauthorizedSites bool + locker sync.RWMutex } @@ -85,6 +95,51 @@ func (exchanger *BaseExchanger) IsConnectorEnabled(id string) bool { return false } +// Update is called whenever the mesh data set has changed to reflect these changes. +func (exchanger *BaseExchanger) Update(meshDataSet meshdata.Map) error { + // Update the stored mesh data set + if err := exchanger.storeMeshDataSet(meshDataSet); err != nil { + return fmt.Errorf("unable to store the mesh data: %v", err) + } + + return nil +} + +func (exchanger *BaseExchanger) storeMeshDataSet(meshDataSet meshdata.Map) error { + // Store the new mesh data set by cloning it and then merging the cloned data into one object + meshDataSetCloned := make(meshdata.Map) + for connectorID, meshData := range meshDataSet { + if !exchanger.IsConnectorEnabled(connectorID) { + continue + } + + meshDataCloned := meshData.Clone() + if meshDataCloned == nil { + return fmt.Errorf("unable to clone the mesh data") + } + + if !exchanger.allowUnauthorizedSites { + exchanger.removeUnauthorizedSites(meshDataCloned) + } + + meshDataSetCloned[connectorID] = meshDataCloned + } + exchanger.setMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) + + return nil +} + +func (exchanger *BaseExchanger) removeUnauthorizedSites(meshData *meshdata.MeshData) { + cleanedSites := make([]*meshdata.Site, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + // Only keep authorized sites + if value := meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false"); strings.EqualFold(value, "true") { + cleanedSites = append(cleanedSites, site) + } + } + meshData.Sites = cleanedSites +} + // Config returns the configuration object. func (exchanger *BaseExchanger) Config() *config.Configuration { return exchanger.conf @@ -105,6 +160,23 @@ func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) { exchanger.enabledConnectors = connectors } +// MeshData returns the stored mesh data. +func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { + return exchanger.meshData +} + +func (exchanger *BaseExchanger) setMeshData(meshData *meshdata.MeshData) { + exchanger.locker.Lock() + defer exchanger.locker.Unlock() + + exchanger.meshData = meshData +} + +// SetAllowUnauthorizedSites sets whether this exchanger allows the exchange of unauthorized sites. +func (exchanger *BaseExchanger) SetAllowUnauthorizedSites(allow bool) { + exchanger.allowUnauthorizedSites = allow +} + // Locker returns the locking object. func (exchanger *BaseExchanger) Locker() *sync.RWMutex { return &exchanger.locker diff --git a/pkg/mentix/exchangers/exporters/exporter.go b/pkg/mentix/exchangers/exporters/exporter.go index 605ca083f2..2b25d7228e 100755 --- a/pkg/mentix/exchangers/exporters/exporter.go +++ b/pkg/mentix/exchangers/exporters/exporter.go @@ -19,87 +19,15 @@ package exporters import ( - "fmt" - "strings" - "github.com/cs3org/reva/pkg/mentix/exchangers" - "github.com/cs3org/reva/pkg/mentix/meshdata" ) // Exporter is the interface that all exporters must implement. type Exporter interface { exchangers.Exchanger - - // MeshData returns the mesh data. - MeshData() *meshdata.MeshData - - // Update is called whenever the mesh data set has changed to reflect these changes. - Update(meshdata.Map) error } // BaseExporter implements basic exporter functionality common to all exporters. type BaseExporter struct { exchangers.BaseExchanger - - meshData *meshdata.MeshData - - allowUnauthorizedSites bool -} - -// Update is called whenever the mesh data set has changed to reflect these changes. -func (exporter *BaseExporter) Update(meshDataSet meshdata.Map) error { - // Update the stored mesh data set - if err := exporter.storeMeshDataSet(meshDataSet); err != nil { - return fmt.Errorf("unable to store the mesh data: %v", err) - } - - return nil -} - -func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.Map) error { - // Store the new mesh data set by cloning it and then merging the cloned data into one object - meshDataSetCloned := make(meshdata.Map) - for connectorID, meshData := range meshDataSet { - if !exporter.IsConnectorEnabled(connectorID) { - continue - } - - meshDataCloned := meshData.Clone() - if meshDataCloned == nil { - return fmt.Errorf("unable to clone the mesh data") - } - - if !exporter.allowUnauthorizedSites { - exporter.removeUnauthorizedSites(meshDataCloned) - } - - meshDataSetCloned[connectorID] = meshDataCloned - } - exporter.SetMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) - - return nil -} - -// MeshData returns the stored mesh data. -func (exporter *BaseExporter) MeshData() *meshdata.MeshData { - return exporter.meshData -} - -// SetMeshData sets new mesh data. -func (exporter *BaseExporter) SetMeshData(meshData *meshdata.MeshData) { - exporter.Locker().Lock() - defer exporter.Locker().Unlock() - - exporter.meshData = meshData -} - -func (exporter *BaseExporter) removeUnauthorizedSites(meshData *meshdata.MeshData) { - cleanedSites := make([]*meshdata.Site, 0, len(meshData.Sites)) - for _, site := range meshData.Sites { - // Only keep authorized sites - if value := meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false"); strings.EqualFold(value, "true") { - cleanedSites = append(cleanedSites, site) - } - } - meshData.Sites = cleanedSites } diff --git a/pkg/mentix/exchangers/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go index cb7c59965f..e65b00c662 100755 --- a/pkg/mentix/exchangers/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -39,8 +39,7 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo // Store WebAPI specifics exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint, conf.Exporters.WebAPI.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) - - exporter.allowUnauthorizedSites = true + exporter.SetAllowUnauthorizedSites(true) exporter.RegisterActionHandler("", webapi.HandleDefaultQuery) diff --git a/pkg/mentix/exchangers/importers/importer.go b/pkg/mentix/exchangers/importers/importer.go index e6625bc868..07c3cc2811 100755 --- a/pkg/mentix/exchangers/importers/importer.go +++ b/pkg/mentix/exchangers/importers/importer.go @@ -21,6 +21,7 @@ package importers import ( "fmt" "strings" + "sync" "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/exchangers" @@ -31,8 +32,8 @@ import ( type Importer interface { exchangers.Exchanger - // MeshData returns the vector of imported mesh data. - MeshData() meshdata.Vector + // MeshDataUpdates returns the vector of imported mesh data. + MeshDataUpdates() meshdata.Vector // Process is called periodically to perform the actual import; if data has been imported, true is returned. Process(*connectors.Collection) (bool, error) @@ -42,31 +43,33 @@ type Importer interface { type BaseImporter struct { exchangers.BaseExchanger - meshData meshdata.Vector + meshDataUpdates meshdata.Vector + + updatesLocker sync.RWMutex } // Process is called periodically to perform the actual import; if data has been imported, true is returned. func (importer *BaseImporter) Process(connectors *connectors.Collection) (bool, error) { - if importer.meshData == nil { // No data present for updating, so nothing to process + if importer.meshDataUpdates == nil { // No data present for updating, so nothing to process return false, nil } var processErrs []string // Data is read, so lock it for writing during the loop - importer.Locker().RLock() + importer.updatesLocker.RLock() for _, connector := range connectors.Connectors { if !importer.IsConnectorEnabled(connector.GetID()) { continue } - if err := importer.processMeshData(connector); err != nil { + if err := importer.processMeshDataUpdates(connector); err != nil { processErrs = append(processErrs, fmt.Sprintf("unable to process imported mesh data for connector '%v': %v", connector.GetName(), err)) } } - importer.Locker().RUnlock() + importer.updatesLocker.RUnlock() - importer.SetMeshData(nil) + importer.setMeshDataUpdates(nil) var err error if len(processErrs) != 0 { @@ -75,8 +78,8 @@ func (importer *BaseImporter) Process(connectors *connectors.Collection) (bool, return true, err } -func (importer *BaseImporter) processMeshData(connector connectors.Connector) error { - for _, meshData := range importer.meshData { +func (importer *BaseImporter) processMeshDataUpdates(connector connectors.Connector) error { + for _, meshData := range importer.meshDataUpdates { if err := connector.UpdateMeshData(meshData); err != nil { return fmt.Errorf("error while updating mesh data: %v", err) } @@ -86,14 +89,13 @@ func (importer *BaseImporter) processMeshData(connector connectors.Connector) er } // MeshData returns the vector of imported mesh data. -func (importer *BaseImporter) MeshData() meshdata.Vector { - return importer.meshData +func (importer *BaseImporter) MeshDataUpdates() meshdata.Vector { + return importer.meshDataUpdates } -// SetMeshData sets the new mesh data vector. -func (importer *BaseImporter) SetMeshData(meshData meshdata.Vector) { - importer.Locker().Lock() - defer importer.Locker().Unlock() +func (importer *BaseImporter) setMeshDataUpdates(meshDataUpdates meshdata.Vector) { + importer.updatesLocker.Lock() + defer importer.updatesLocker.Unlock() - importer.meshData = meshData + importer.meshDataUpdates = meshDataUpdates } diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index d229de834d..457bf320ba 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -50,14 +50,14 @@ func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req func (importer *BaseRequestImporter) mergeImportedMeshDataSet(meshDataSet meshdata.Vector) { // Merge the newly imported data with any existing data stored in the importer - if importer.meshData != nil { + if importer.meshDataUpdates != nil { // Need to manually lock the data for writing - importer.Locker().Lock() - defer importer.Locker().Unlock() + importer.updatesLocker.Lock() + defer importer.updatesLocker.Unlock() - importer.meshData = append(importer.meshData, meshDataSet...) + importer.meshDataUpdates = append(importer.meshDataUpdates, meshDataSet...) } else { - importer.SetMeshData(meshDataSet) // SetMeshData will do the locking itself + importer.setMeshDataUpdates(meshDataSet) // SetMeshData will do the locking itself } } diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go index 90de14195e..8bc92ee68f 100644 --- a/pkg/mentix/exchangers/importers/sitereg.go +++ b/pkg/mentix/exchangers/importers/sitereg.go @@ -44,6 +44,7 @@ func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, l // Store SiteRegistration specifics importer.SetEndpoint(conf.Importers.SiteRegistration.Endpoint, conf.Importers.SiteRegistration.IsProtected) importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) + importer.SetAllowUnauthorizedSites(true) importer.RegisterActionHandler("register", sitereg.HandleRegisterSiteQuery) importer.RegisterActionHandler("unregister", sitereg.HandleUnregisterSiteQuery) diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index 7dac4302e4..ded3c73431 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -92,8 +92,6 @@ func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Value return createErrorResponse("INVALID_API_KEY", err) } - // TODO: Check if site with ID already exists; bail out if so (or update, whatever) - // Decode the site registration data and convert it to a meshdata object siteData, err := decodeQueryData(data) if err != nil { diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 684cc10106..37125c2793 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -267,9 +267,13 @@ func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error { mntx.meshDataSet = meshDataSet - for _, exporter := range mntx.exporters.Exporters { - if err := exporter.Update(mntx.meshDataSet); err != nil { - return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) + exchangers := make([]exchangers.Exchanger, 0, len(mntx.exporters.Exporters)+len(mntx.importers.Importers)) + exchangers = append(exchangers, mntx.exporters.Exchangers()...) + exchangers = append(exchangers, mntx.importers.Exchangers()...) + + for _, exchanger := range exchangers { + if err := exchanger.Update(mntx.meshDataSet); err != nil { + return fmt.Errorf("unable to update mesh data on exchanger '%v': %v", exchanger.GetName(), err) } } } From 5d1ff52dc003afde2a7ce57451975c5ad84d91cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 12:28:50 +0100 Subject: [PATCH 41/59] Check site existence on operations --- .../exchangers/importers/reqimporter.go | 6 +++- .../exchangers/importers/sitereg/query.go | 32 ++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 457bf320ba..34c79d5c89 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -62,5 +62,9 @@ func (importer *BaseRequestImporter) mergeImportedMeshDataSet(meshDataSet meshda } func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - return importer.HandleAction(nil, data, params) + // Data is read, so lock it for writing + importer.Locker().RLock() + defer importer.Locker().RUnlock() + + return importer.HandleAction(importer.MeshData(), data, params) } diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index ded3c73431..0096f979c8 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -86,7 +86,7 @@ func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, e } // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { siteID, flags, err := decodeAPIKey(params) if err != nil { return createErrorResponse("INVALID_API_KEY", err) @@ -108,34 +108,42 @@ func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Value return createErrorResponse("INVALID_SITE_DATA", err) } - meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - if err := meshData.Verify(); err != nil { + meshDataUpdate := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + if err := meshDataUpdate.Verify(); err != nil { return createErrorResponse("INVALID_MESH_DATA", err) } - meshData.Status = meshdata.StatusDefault - meshData.InferMissingData() + meshDataUpdate.Status = meshdata.StatusDefault + meshDataUpdate.InferMissingData() - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse("SITE_REGISTERED", network.ResponseParams{"id": siteID}), nil + msg := "SITE_REGISTERED" + if meshData.FindSite(siteID) != nil { + msg = "SITE_UPDATED" + } + + return meshdata.Vector{meshDataUpdate}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": siteID}), nil } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(_ *meshdata.MeshData, _ []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values) (meshdata.Vector, int, []byte, error) { siteID, _, err := decodeAPIKey(params) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } - // TODO: Check if site with ID exists; bail out if not - // The site ID must be provided in the call as well to enhance security further if params.Get("siteId") != siteID { return createErrorResponse("INVALID_SITE_ID", errors.Errorf("site ID mismatch")) } + // Check if the site to be removed actually exists + if meshData.FindSite(siteID) == nil { + return createErrorResponse("INVALID_SITE_ID", errors.Errorf("site not found")) + } + // To remove a site, a meshdata object that contains a site with the given ID needs to be created site := &meshdata.Site{ID: siteID} - meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - meshData.Status = meshdata.StatusObsolete + meshDataUpdate := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + meshDataUpdate.Status = meshdata.StatusObsolete - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse("SITE_UNREGISTERED", network.ResponseParams{"id": siteID}), nil + return meshdata.Vector{meshDataUpdate}, http.StatusOK, network.CreateResponse("SITE_UNREGISTERED", network.ResponseParams{"id": siteID}), nil } From 4d4a1c316299b434783a067e5e53938b012ec861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 12:47:42 +0100 Subject: [PATCH 42/59] Pass config and log to request exchangers --- pkg/mentix/config/config.go | 7 +++-- .../exchangers/exporters/cs3api/query.go | 4 ++- .../exchangers/exporters/reqexporter.go | 11 ++++--- .../exchangers/exporters/siteloc/query.go | 5 +++- .../exchangers/exporters/webapi/query.go | 5 +++- .../exchangers/importers/reqimporter.go | 11 ++++--- pkg/mentix/exchangers/importers/sitereg.go | 5 ---- .../importers/sitereg/accservice.go | 30 ++++--------------- .../exchangers/importers/sitereg/query.go | 14 +++++---- .../exchangers/importers/webapi/query.go | 7 +++-- pkg/mentix/exchangers/reqexchanger.go | 13 ++++---- pkg/mentix/mentix.go | 2 +- 12 files changed, 57 insertions(+), 57 deletions(-) diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 639140394e..8068ddc913 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -43,9 +43,10 @@ type Configuration struct { } `mapstructure:"webapi"` SiteRegistration struct { - Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` - EnabledConnectors []string `mapstructure:"enabled_connectors"` + Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` + IgnoreScienceMeshSites bool `mapstructure:"ignore_sm_sites"` } `mapstructure:"sitereg"` } `mapstructure:"importers"` diff --git a/pkg/mentix/exchangers/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go index 9859953bfc..adc923f6e9 100755 --- a/pkg/mentix/exchangers/exporters/cs3api/query.go +++ b/pkg/mentix/exchangers/exporters/cs3api/query.go @@ -25,12 +25,14 @@ import ( "net/url" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { diff --git a/pkg/mentix/exchangers/exporters/reqexporter.go b/pkg/mentix/exchangers/exporters/reqexporter.go index 42cb274b11..0e34a6de5d 100644 --- a/pkg/mentix/exchangers/exporters/reqexporter.go +++ b/pkg/mentix/exchangers/exporters/reqexporter.go @@ -23,6 +23,9 @@ import ( "net/http" "net/url" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchangers" ) @@ -33,9 +36,9 @@ type BaseRequestExporter struct { } // HandleRequest handles the actual HTTP request. -func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { +func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) { body, _ := ioutil.ReadAll(req.Body) - status, respData, err := exporter.handleQuery(body, req.URL.Query()) + status, respData, err := exporter.handleQuery(body, req.URL.Query(), conf, log) if err != nil { respData = []byte(err.Error()) } @@ -43,11 +46,11 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req _, _ = resp.Write(respData) } -func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values) (int, []byte, error) { +func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (int, []byte, error) { // Data is read, so lock it for writing exporter.Locker().RLock() defer exporter.Locker().RUnlock() - _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params) + _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params, conf, log) return status, data, err } diff --git a/pkg/mentix/exchangers/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go index 09aa4f985b..db5243a272 100755 --- a/pkg/mentix/exchangers/exporters/siteloc/query.go +++ b/pkg/mentix/exchangers/exporters/siteloc/query.go @@ -24,11 +24,14 @@ import ( "net/http" "net/url" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { // Convert the mesh data locData, err := convertMeshDataToLocationData(meshData) if err != nil { diff --git a/pkg/mentix/exchangers/exporters/webapi/query.go b/pkg/mentix/exchangers/exporters/webapi/query.go index e4886b91bb..4f596e78a9 100755 --- a/pkg/mentix/exchangers/exporters/webapi/query.go +++ b/pkg/mentix/exchangers/exporters/webapi/query.go @@ -24,11 +24,14 @@ import ( "net/http" "net/url" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 34c79d5c89..abb5599481 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -23,6 +23,9 @@ import ( "net/http" "net/url" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -34,9 +37,9 @@ type BaseRequestImporter struct { } // HandleRequest handles the actual HTTP request. -func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { +func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) { body, _ := ioutil.ReadAll(req.Body) - meshDataSet, status, respData, err := importer.handleQuery(body, req.URL.Query()) + meshDataSet, status, respData, err := importer.handleQuery(body, req.URL.Query(), conf, log) if err == nil { if len(meshDataSet) > 0 { importer.mergeImportedMeshDataSet(meshDataSet) @@ -61,10 +64,10 @@ func (importer *BaseRequestImporter) mergeImportedMeshDataSet(meshDataSet meshda } } -func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) { // Data is read, so lock it for writing importer.Locker().RLock() defer importer.Locker().RUnlock() - return importer.HandleAction(importer.MeshData(), data, params) + return importer.HandleAction(importer.MeshData(), data, params, conf, log) } diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go index 8bc92ee68f..8c1af735c7 100644 --- a/pkg/mentix/exchangers/importers/sitereg.go +++ b/pkg/mentix/exchangers/importers/sitereg.go @@ -19,7 +19,6 @@ package importers import ( - "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix/config" @@ -37,10 +36,6 @@ func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, l return err } - if err := sitereg.ConfigureAccountsService(conf.AccountsService.URL, conf.AccountsService.User, conf.AccountsService.Password); err != nil { - return errors.Wrap(err, "error while configuring the accounts service") - } - // Store SiteRegistration specifics importer.SetEndpoint(conf.Importers.SiteRegistration.Endpoint, conf.Importers.SiteRegistration.IsProtected) importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) diff --git a/pkg/mentix/exchangers/importers/sitereg/accservice.go b/pkg/mentix/exchangers/importers/sitereg/accservice.go index 7f670d9725..dcf9ae84ae 100644 --- a/pkg/mentix/exchangers/importers/sitereg/accservice.go +++ b/pkg/mentix/exchangers/importers/sitereg/accservice.go @@ -27,46 +27,28 @@ import ( "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/utils/network" ) -type accountsServiceSettings struct { - URL *url.URL - user string - password string -} - type requestResponse struct { Success bool Error string Data interface{} } -var ( - accountsService accountsServiceSettings -) - -// ConfigureAccountsService configures the accounts service for the site registration importer. -func ConfigureAccountsService(address string, user string, password string) error { - endpointURL, err := url.Parse(address) +func queryAccountsService(endpoint string, params network.URLParams, conf *config.Configuration) (*requestResponse, error) { + URL, err := url.Parse(conf.AccountsService.URL) if err != nil { - return errors.Wrap(err, "unable to parse the accounts service URL") + return nil, errors.Wrap(err, "unable to parse the accounts service URL") } - accountsService.URL = endpointURL - accountsService.user = user - accountsService.password = password - - return nil -} - -func queryAccountsService(endpoint string, params network.URLParams) (*requestResponse, error) { - fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", accountsService.URL.Scheme, accountsService.URL.Host), path.Join(accountsService.URL.Path, endpoint), params) + fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", URL.Scheme, URL.Host), path.Join(URL.Path, endpoint), params) if err != nil { return nil, errors.Wrap(err, "error while building the service accounts query URL") } - data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: accountsService.user, Password: accountsService.password}, false) + data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: conf.AccountsService.User, Password: conf.AccountsService.Password}, false) if err != nil { return nil, errors.Wrap(err, "unable to query the service accounts endpoint") } diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index 0096f979c8..bb15fc3a6c 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -25,7 +25,9 @@ import ( "strings" "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/mentix/meshdata" "github.com/cs3org/reva/pkg/mentix/utils/network" @@ -44,14 +46,14 @@ func decodeQueryData(data []byte) (*siteRegistrationData, error) { return siteData, nil } -func decodeAPIKey(params url.Values) (key.SiteIdentifier, int8, error) { +func decodeAPIKey(params url.Values, conf *config.Configuration) (key.SiteIdentifier, int8, error) { apiKey := params.Get("apiKey") if len(apiKey) == 0 { return "", 0, errors.Errorf("no API key specified") } // Try to get an account that is associated with the given API key; if none exists, return an error - resp, err := queryAccountsService("find", network.URLParams{"by": "apikey", "value": apiKey}) + resp, err := queryAccountsService("find", network.URLParams{"by": "apikey", "value": apiKey}, conf) if err != nil { return "", 0, errors.Wrap(err, "error while querying the accounts service") } @@ -86,8 +88,8 @@ func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, e } // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - siteID, flags, err := decodeAPIKey(params) +func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { + siteID, flags, err := decodeAPIKey(params, conf) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } @@ -124,8 +126,8 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - siteID, _, err := decodeAPIKey(params) +func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { + siteID, _, err := decodeAPIKey(params, conf) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index 19e1dca988..a92407511a 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -24,6 +24,9 @@ import ( "net/http" "net/url" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" "github.com/cs3org/reva/pkg/mentix/utils/network" ) @@ -59,11 +62,11 @@ func handleQuery(data []byte, params url.Values, status int, msg string) (meshda } // HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func HandleUnregisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/exchangers/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go index d5e21eb75e..cdee85d915 100644 --- a/pkg/mentix/exchangers/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -24,6 +24,9 @@ import ( "net/url" "strings" + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -36,10 +39,10 @@ type RequestExchanger interface { // WantsRequest returns whether the exchanger wants to handle the incoming request. WantsRequest(r *http.Request) bool // HandleRequest handles the actual HTTP request. - HandleRequest(resp http.ResponseWriter, req *http.Request) + HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) } -type queryCallback func(*meshdata.MeshData, []byte, url.Values) (meshdata.Vector, int, []byte, error) +type queryCallback func(*meshdata.MeshData, []byte, url.Values, *config.Configuration, *zerolog.Logger) (meshdata.Vector, int, []byte, error) // BaseRequestExchanger implements basic exporter functionality common to all request exporters. type BaseRequestExchanger struct { @@ -78,7 +81,7 @@ func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool { } // HandleRequest handles the actual HTTP request. -func (exchanger *BaseRequestExchanger) HandleRequest(resp http.ResponseWriter, req *http.Request) error { +func (exchanger *BaseRequestExchanger) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) error { return nil } @@ -91,11 +94,11 @@ func (exchanger *BaseRequestExchanger) RegisterActionHandler(action string, call } // HandleAction executes the registered handler for the specified action, if any. -func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) { reqAction := params.Get("action") for action, handler := range exchanger.actionHandlers { if strings.EqualFold(action, reqAction) { - return handler(meshData, body, params) + return handler(meshData, body, params, conf, log) } } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 37125c2793..a8a73d37ba 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -314,7 +314,7 @@ func (mntx *Mentix) handleRequest(exchangers []exchangers.RequestExchanger, w ht // Ask each RequestExchanger if it wants to handle the request for _, exchanger := range exchangers { if exchanger.WantsRequest(r) { - exchanger.HandleRequest(w, r) + exchanger.HandleRequest(w, r, mntx.conf, log) } } } From 10a464dbf089fa6ae3d73cb155d96f345c62bb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 12:53:12 +0100 Subject: [PATCH 43/59] Add email to registered sites --- .../exchangers/importers/sitereg/query.go | 22 +++++++++---------- .../exchangers/importers/sitereg/types.go | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index bb15fc3a6c..94b5a9f8df 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -46,19 +46,19 @@ func decodeQueryData(data []byte) (*siteRegistrationData, error) { return siteData, nil } -func decodeAPIKey(params url.Values, conf *config.Configuration) (key.SiteIdentifier, int8, error) { +func extractQueryInformation(params url.Values, conf *config.Configuration) (key.SiteIdentifier, int8, string, error) { apiKey := params.Get("apiKey") if len(apiKey) == 0 { - return "", 0, errors.Errorf("no API key specified") + return "", 0, "", errors.Errorf("no API key specified") } // Try to get an account that is associated with the given API key; if none exists, return an error resp, err := queryAccountsService("find", network.URLParams{"by": "apikey", "value": apiKey}, conf) if err != nil { - return "", 0, errors.Wrap(err, "error while querying the accounts service") + return "", 0, "", errors.Wrap(err, "error while querying the accounts service") } if !resp.Success { - return "", 0, errors.Errorf("unable to fetch account associated with the provided API key: %v", resp.Error) + return "", 0, "", errors.Errorf("unable to fetch account associated with the provided API key: %v", resp.Error) } // Extract email from account data; this is needed to calculate the site ID from the API key @@ -67,20 +67,20 @@ func decodeAPIKey(params url.Values, conf *config.Configuration) (key.SiteIdenti email, _ = value.(string) } if len(email) == 0 { - return "", 0, errors.Errorf("could not get the email address of the user account") + return "", 0, "", errors.Errorf("could not get the email address of the user account") } _, flags, _, err := key.SplitAPIKey(apiKey) if err != nil { - return "", 0, errors.Errorf("sticky API key specified") + return "", 0, "", errors.Errorf("sticky API key specified") } siteID, err := key.CalculateSiteID(apiKey, strings.ToLower(email)) if err != nil { - return "", 0, errors.Wrap(err, "unable to get site ID") + return "", 0, "", errors.Wrap(err, "unable to get site ID") } - return siteID, flags, nil + return siteID, flags, email, nil } func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, error) { @@ -89,7 +89,7 @@ func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, e // HandleRegisterSiteQuery registers a site. func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, flags, err := decodeAPIKey(params, conf) + siteID, flags, email, err := extractQueryInformation(params, conf) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } @@ -105,7 +105,7 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur siteType = meshdata.SiteTypeScienceMesh } - site, err := siteData.ToMeshDataSite(siteID, siteType) + site, err := siteData.ToMeshDataSite(siteID, siteType, email) if err != nil { return createErrorResponse("INVALID_SITE_DATA", err) } @@ -127,7 +127,7 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur // HandleUnregisterSiteQuery unregisters a site. func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, _, err := decodeAPIKey(params, conf) + siteID, _, _, err := extractQueryInformation(params, conf) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } diff --git a/pkg/mentix/exchangers/importers/sitereg/types.go b/pkg/mentix/exchangers/importers/sitereg/types.go index 8061ba6f13..e64fe8e9f0 100644 --- a/pkg/mentix/exchangers/importers/sitereg/types.go +++ b/pkg/mentix/exchangers/importers/sitereg/types.go @@ -82,7 +82,7 @@ func (siteData *siteRegistrationData) Verify() error { } // ToMeshDataSite converts the stored data into a meshdata site object, filling out as much data as possible. -func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, siteType meshdata.SiteType) (*meshdata.Site, error) { +func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, siteType meshdata.SiteType, email string) (*meshdata.Site, error) { siteUrl, err := url.Parse(siteData.URL) if err != nil { return nil, errors.Wrap(err, "invalid site URL") @@ -127,7 +127,7 @@ func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, Organization: "", Domain: network.ExtractDomainFromURL(siteUrl, true), Homepage: siteData.URL, - Email: "", // TODO: Get account data + Email: email, Description: siteData.Name + " @ " + siteData.URL, Country: countries.LookupCountry(siteData.CountryCode), CountryCode: siteData.CountryCode, From c6d3a0b937b8d0b33ada3ec7f9cd0c7fbcf5d0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 13:03:18 +0100 Subject: [PATCH 44/59] Option to ignore registrations of ScienceMesh sites --- pkg/mentix/exchangers/importers/sitereg/query.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index 94b5a9f8df..e6508ec8c1 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -94,6 +94,11 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur return createErrorResponse("INVALID_API_KEY", err) } + msg := "SITE_REGISTERED" + if meshData.FindSite(siteID) != nil { + msg = "SITE_UPDATED" + } + // Decode the site registration data and convert it to a meshdata object siteData, err := decodeQueryData(data) if err != nil { @@ -105,6 +110,11 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur siteType = meshdata.SiteTypeScienceMesh } + // If the corresponding setting is set, ignore registrations of ScienceMesh sites + if siteType == meshdata.SiteTypeScienceMesh && conf.Importers.SiteRegistration.IgnoreScienceMeshSites { + return meshdata.Vector{}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": siteID}), nil + } + site, err := siteData.ToMeshDataSite(siteID, siteType, email) if err != nil { return createErrorResponse("INVALID_SITE_DATA", err) @@ -117,11 +127,6 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur meshDataUpdate.Status = meshdata.StatusDefault meshDataUpdate.InferMissingData() - msg := "SITE_REGISTERED" - if meshData.FindSite(siteID) != nil { - msg = "SITE_UPDATED" - } - return meshdata.Vector{meshDataUpdate}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": siteID}), nil } From fd6cab8aa02f3ae245fd90ace8b14c5cbc9f9e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 13:29:03 +0100 Subject: [PATCH 45/59] Better accounts services initialization --- .../sitereg => accservice}/accservice.go | 36 ++++++++++++++----- .../exchangers/importers/sitereg/query.go | 13 +++---- pkg/mentix/mentix.go | 6 ++++ pkg/mentix/meshdata/site.go | 10 ++++++ 4 files changed, 50 insertions(+), 15 deletions(-) rename pkg/mentix/{exchangers/importers/sitereg => accservice}/accservice.go (67%) diff --git a/pkg/mentix/exchangers/importers/sitereg/accservice.go b/pkg/mentix/accservice/accservice.go similarity index 67% rename from pkg/mentix/exchangers/importers/sitereg/accservice.go rename to pkg/mentix/accservice/accservice.go index dcf9ae84ae..64c5b95061 100644 --- a/pkg/mentix/exchangers/importers/sitereg/accservice.go +++ b/pkg/mentix/accservice/accservice.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package sitereg +package accservice import ( "encoding/json" @@ -37,18 +37,22 @@ type requestResponse struct { Data interface{} } -func queryAccountsService(endpoint string, params network.URLParams, conf *config.Configuration) (*requestResponse, error) { - URL, err := url.Parse(conf.AccountsService.URL) - if err != nil { - return nil, errors.Wrap(err, "unable to parse the accounts service URL") - } +type accountsServiceSettings struct { + URL *url.URL + User string + Password string +} - fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", URL.Scheme, URL.Host), path.Join(URL.Path, endpoint), params) +var settings accountsServiceSettings + +// Query performs an account service query. +func Query(endpoint string, params network.URLParams) (*requestResponse, error) { + fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", settings.URL.Scheme, settings.URL.Host), path.Join(settings.URL.Path, endpoint), params) if err != nil { return nil, errors.Wrap(err, "error while building the service accounts query URL") } - data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: conf.AccountsService.User, Password: conf.AccountsService.Password}, false) + data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: settings.User, Password: settings.Password}, false) if err != nil { return nil, errors.Wrap(err, "unable to query the service accounts endpoint") } @@ -60,7 +64,8 @@ func queryAccountsService(endpoint string, params network.URLParams, conf *confi return resp, nil } -func getResponseValue(resp *requestResponse, path string) interface{} { +// GetResponseValue gets a value from an account service query using a dotted path notation. +func GetResponseValue(resp *requestResponse, path string) interface{} { if data, ok := resp.Data.(map[string]interface{}); ok { tokens := strings.Split(path, ".") for i, name := range tokens { @@ -79,3 +84,16 @@ func getResponseValue(resp *requestResponse, path string) interface{} { return nil } + +func InitAccountsService(conf *config.Configuration) error { + URL, err := url.Parse(conf.AccountsService.URL) + if err != nil { + return errors.Wrap(err, "unable to parse the accounts service URL") + } + + settings.URL = URL + settings.User = conf.AccountsService.User + settings.Password = conf.AccountsService.Password + + return nil +} diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index e6508ec8c1..4a150e40dd 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -30,6 +30,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/utils/accservice" "github.com/cs3org/reva/pkg/mentix/utils/network" ) @@ -46,14 +47,14 @@ func decodeQueryData(data []byte) (*siteRegistrationData, error) { return siteData, nil } -func extractQueryInformation(params url.Values, conf *config.Configuration) (key.SiteIdentifier, int8, string, error) { +func extractQueryInformation(params url.Values) (key.SiteIdentifier, int8, string, error) { apiKey := params.Get("apiKey") if len(apiKey) == 0 { return "", 0, "", errors.Errorf("no API key specified") } // Try to get an account that is associated with the given API key; if none exists, return an error - resp, err := queryAccountsService("find", network.URLParams{"by": "apikey", "value": apiKey}, conf) + resp, err := accservice.Query("find", network.URLParams{"by": "apikey", "value": apiKey}) if err != nil { return "", 0, "", errors.Wrap(err, "error while querying the accounts service") } @@ -63,7 +64,7 @@ func extractQueryInformation(params url.Values, conf *config.Configuration) (key // Extract email from account data; this is needed to calculate the site ID from the API key email := "" - if value := getResponseValue(resp, "account.email"); value != nil { + if value := accservice.GetResponseValue(resp, "account.email"); value != nil { email, _ = value.(string) } if len(email) == 0 { @@ -89,7 +90,7 @@ func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, e // HandleRegisterSiteQuery registers a site. func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, flags, email, err := extractQueryInformation(params, conf) + siteID, flags, email, err := extractQueryInformation(params) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } @@ -131,8 +132,8 @@ func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params ur } // HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, _, _, err := extractQueryInformation(params, conf) +func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { + siteID, _, _, err := extractQueryInformation(params) if err != nil { return createErrorResponse("INVALID_API_KEY", err) } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index a8a73d37ba..afc641c9e0 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/exchangers/exporters" "github.com/cs3org/reva/pkg/mentix/exchangers/importers" "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/utils/accservice" ) // Mentix represents the main Mentix service object. @@ -321,6 +322,11 @@ func (mntx *Mentix) handleRequest(exchangers []exchangers.RequestExchanger, w ht // New creates a new Mentix service instance. func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) { + // Configure the accounts service upfront + if err := accservice.InitAccountsService(conf); err != nil { + return nil, fmt.Errorf("unable to initialize the accounts service: %v", err) + } + mntx := new(Mentix) if err := mntx.initialize(conf, log); err != nil { return nil, fmt.Errorf("unable to initialize Mentix: %v", err) diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 35d9cc0248..ec36805ae6 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -129,6 +129,16 @@ func (site *Site) InferMissingData() { } } +func (site *Site) IsAuthorized() bool { + // ScienceMesh sites are always authorized + if site.Type == SiteTypeScienceMesh { + return true + } + + // TODO: Use accounts service + return false +} + // GetSiteTypeName returns the readable name of the given site type. func GetSiteTypeName(siteType SiteType) string { switch siteType { From d6e78810162e8e2b15a4390f0003262a0a492b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 13:37:54 +0100 Subject: [PATCH 46/59] Extend is-authorized endpoint --- internal/http/services/accounts/accounts.go | 10 ++++++---- internal/http/services/accounts/manager.go | 18 ++++++++++++++++++ .../exchangers/importers/sitereg/query.go | 2 +- pkg/mentix/mentix.go | 2 +- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 89796a3e7a..ea72de7393 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -63,6 +63,7 @@ func (s *svc) Prefix() string { // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { + return []string{"/"} // This service currently only has one public endpoint (called "register") used for account registration return []string{config.EndpointCreate} } @@ -272,14 +273,15 @@ func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) } func (s *svc) handleIsAuthorized(values url.Values, body []byte) (interface{}, error) { - apiKey := values.Get("apiKey") + by := values.Get("by") + value := values.Get("value") - if len(apiKey) == 0 { - return nil, errors.Errorf("no API key specified") + if len(by) == 0 && len(value) == 0 { + return nil, errors.Errorf("missing search criteria") } // Find the account associated with the given API key - account, err := s.manager.FindAccount(FindByAPIKey, apiKey) + account, err := s.manager.FindAccount(by, value) if err != nil { return nil, errors.Wrap(err, "no user with the specified API key found") } diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 3175f05839..afb3cec27d 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -40,6 +40,7 @@ import ( const ( FindByEmail = "email" FindByAPIKey = "apikey" + FindBySiteID = "siteid" ) // Manager is responsible for all user account related tasks. @@ -144,6 +145,20 @@ func (mngr *Manager) findAccountByAPIKey(key key.APIKey) *data.Account { return nil } +func (mngr *Manager) findAccountBySiteID(siteID key.SiteIdentifier) *data.Account { + if siteID == "" { + return nil + } + + // Perform a case-sensitive search of the given site ID + for _, account := range mngr.accounts { + if account.GetSiteID() == siteID { + return account + } + } + return nil +} + func (mngr *Manager) ShowPanel(w http.ResponseWriter) error { // The panel only shows the stored accounts and offers actions through links, so let it use cloned data accounts := mngr.CloneAccounts() @@ -207,6 +222,9 @@ func (mngr *Manager) FindAccount(by string, value string) (*data.Account, error) case FindByAPIKey: account = mngr.findAccountByAPIKey(value) + case FindBySiteID: + account = mngr.findAccountBySiteID(value) + default: return nil, errors.Errorf("invalid search type %v", by) } diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index 4a150e40dd..e542359d9a 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -27,10 +27,10 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/accservice" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/utils/accservice" "github.com/cs3org/reva/pkg/mentix/utils/network" ) diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index afc641c9e0..cb499ae870 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -27,6 +27,7 @@ import ( "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/mentix/accservice" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" "github.com/cs3org/reva/pkg/mentix/entity" @@ -34,7 +35,6 @@ import ( "github.com/cs3org/reva/pkg/mentix/exchangers/exporters" "github.com/cs3org/reva/pkg/mentix/exchangers/importers" "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/utils/accservice" ) // Mentix represents the main Mentix service object. From 78a98d491f1afbd205a4130bf7df94ce753a7c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 14:04:57 +0100 Subject: [PATCH 47/59] Add site authorization check --- internal/http/services/accounts/accounts.go | 1 - pkg/mentix/connectors/localfile.go | 2 +- pkg/mentix/meshdata/site.go | 10 +++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index ea72de7393..2a475d45bc 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -63,7 +63,6 @@ func (s *svc) Prefix() string { // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { - return []string{"/"} // This service currently only has one public endpoint (called "register") used for account registration return []string{config.EndpointCreate} } diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index a3c57d5259..fac2822b11 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -75,7 +75,7 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err return nil, fmt.Errorf("invalid file '%v': %v", connector.filePath, err) } - // Update the site types, as these are not part of the JSON data + // Enforce site types connector.setSiteTypes(meshData) meshData.InferMissingData() diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index ec36805ae6..977e341eaa 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -23,6 +23,7 @@ import ( "net/url" "strings" + "github.com/cs3org/reva/pkg/mentix/accservice" "github.com/cs3org/reva/pkg/mentix/utils/network" ) @@ -135,7 +136,14 @@ func (site *Site) IsAuthorized() bool { return true } - // TODO: Use accounts service + // Use the accounts service to find out whether the site is authorized + resp, err := accservice.Query("is-authorized", network.URLParams{"by": "siteid", "value": site.ID}) + if err == nil && resp.Success { + if authorized, ok := resp.Data.(bool); ok { + return authorized + } + } + return false } From e3e1e04c497551221c01046b2a6ce2c0099b1357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 14:30:05 +0100 Subject: [PATCH 48/59] Check authorization status during all requests --- pkg/mentix/connectors/gocdb.go | 5 ---- pkg/mentix/connectors/localfile.go | 13 --------- pkg/mentix/exchangers/exchanger.go | 28 +++++++++++-------- .../exchangers/exporters/reqexporter.go | 4 --- pkg/mentix/exchangers/importers/importer.go | 8 ------ .../exchangers/importers/webapi/query.go | 3 -- pkg/mentix/meshdata/properties.go | 2 -- 7 files changed, 16 insertions(+), 47 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 1ac667dc9d..ca24e85e71 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -133,11 +133,6 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { return fmt.Errorf("site ID missing for site '%v'", site.ShortName) } - // Sites coming from the GOCDB are always authorized by default - if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 { - meshdata.SetPropertyValue(&properties, meshdata.PropertyAuthorized, "true") - } - // See if an organization has been defined using properties; otherwise, use the official name organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index fac2822b11..6b83aab568 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -114,21 +114,8 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa } func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { - // Store the previous authorization status for already existing sites - siteAuthorizationStatus := make(map[string]string) - for _, site := range meshData.Sites { - siteAuthorizationStatus[site.ID] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") - } - // Add/update data by merging meshData.Merge(updatedData) - - // Restore the authorization status for all sites - for siteID, status := range siteAuthorizationStatus { - if site := meshData.FindSite(siteID); site != nil { - meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, status) - } - } return nil } diff --git a/pkg/mentix/exchangers/exchanger.go b/pkg/mentix/exchangers/exchanger.go index a017f501cb..7f0fa33971 100644 --- a/pkg/mentix/exchangers/exchanger.go +++ b/pkg/mentix/exchangers/exchanger.go @@ -118,10 +118,6 @@ func (exchanger *BaseExchanger) storeMeshDataSet(meshDataSet meshdata.Map) error return fmt.Errorf("unable to clone the mesh data") } - if !exchanger.allowUnauthorizedSites { - exchanger.removeUnauthorizedSites(meshDataCloned) - } - meshDataSetCloned[connectorID] = meshDataCloned } exchanger.setMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) @@ -129,15 +125,23 @@ func (exchanger *BaseExchanger) storeMeshDataSet(meshDataSet meshdata.Map) error return nil } -func (exchanger *BaseExchanger) removeUnauthorizedSites(meshData *meshdata.MeshData) { - cleanedSites := make([]*meshdata.Site, 0, len(meshData.Sites)) - for _, site := range meshData.Sites { - // Only keep authorized sites - if value := meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false"); strings.EqualFold(value, "true") { - cleanedSites = append(cleanedSites, site) +func (exchanger *BaseExchanger) cloneMeshData(clean bool) *meshdata.MeshData { + exchanger.locker.RLock() + meshDataClone := exchanger.meshData.Clone() + exchanger.locker.RUnlock() + + if clean && !exchanger.allowUnauthorizedSites { + cleanedSites := make([]*meshdata.Site, 0, len(meshDataClone.Sites)) + for _, site := range meshDataClone.Sites { + // Only keep authorized sites + if site.IsAuthorized() { + cleanedSites = append(cleanedSites, site) + } } + meshDataClone.Sites = cleanedSites } - meshData.Sites = cleanedSites + + return meshDataClone } // Config returns the configuration object. @@ -162,7 +166,7 @@ func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) { // MeshData returns the stored mesh data. func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { - return exchanger.meshData + return exchanger.cloneMeshData(true) } func (exchanger *BaseExchanger) setMeshData(meshData *meshdata.MeshData) { diff --git a/pkg/mentix/exchangers/exporters/reqexporter.go b/pkg/mentix/exchangers/exporters/reqexporter.go index 0e34a6de5d..ac03f1ec59 100644 --- a/pkg/mentix/exchangers/exporters/reqexporter.go +++ b/pkg/mentix/exchangers/exporters/reqexporter.go @@ -47,10 +47,6 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req } func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (int, []byte, error) { - // Data is read, so lock it for writing - exporter.Locker().RLock() - defer exporter.Locker().RUnlock() - _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params, conf, log) return status, data, err } diff --git a/pkg/mentix/exchangers/importers/importer.go b/pkg/mentix/exchangers/importers/importer.go index 07c3cc2811..d955f3608a 100755 --- a/pkg/mentix/exchangers/importers/importer.go +++ b/pkg/mentix/exchangers/importers/importer.go @@ -32,9 +32,6 @@ import ( type Importer interface { exchangers.Exchanger - // MeshDataUpdates returns the vector of imported mesh data. - MeshDataUpdates() meshdata.Vector - // Process is called periodically to perform the actual import; if data has been imported, true is returned. Process(*connectors.Collection) (bool, error) } @@ -88,11 +85,6 @@ func (importer *BaseImporter) processMeshDataUpdates(connector connectors.Connec return nil } -// MeshData returns the vector of imported mesh data. -func (importer *BaseImporter) MeshDataUpdates() meshdata.Vector { - return importer.meshDataUpdates -} - func (importer *BaseImporter) setMeshDataUpdates(meshDataUpdates meshdata.Vector) { importer.updatesLocker.Lock() defer importer.updatesLocker.Unlock() diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index a92407511a..1221e60d77 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -40,9 +40,6 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { // Imported sites will be assigned an ID automatically site.ID = "" - // Set sites imported through the WebAPI to 'unauthorized' by default - meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") - meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} if err := meshData.Verify(); err != nil { return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 3538c83198..0ca488c997 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -23,8 +23,6 @@ import "strings" const ( // PropertySiteID identifies the site ID property. PropertySiteID = "site_id" - // PropertyAuthorized identifies the authorization status property. - PropertyAuthorized = "authorized" // PropertyOrganization identifies the organization property. PropertyOrganization = "organization" // PropertyMetricsPath identifies the metrics path property. From 5ef9f6a353191e4e592ab3186cd84a00022118ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 14:31:26 +0100 Subject: [PATCH 49/59] Remove WebAPI importer --- internal/http/services/mentix/mentix.go | 4 -- pkg/mentix/config/config.go | 6 -- pkg/mentix/config/ids.go | 2 - pkg/mentix/exchangers/importers/webapi.go | 61 ---------------- .../exchangers/importers/webapi/query.go | 69 ------------------- 5 files changed, 142 deletions(-) delete mode 100755 pkg/mentix/exchangers/importers/webapi.go delete mode 100755 pkg/mentix/exchangers/importers/webapi/query.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index b33548ae54..2d82bdb88e 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -137,10 +137,6 @@ func applyDefaultConfig(conf *config.Configuration) { } // Importers - if conf.Importers.WebAPI.Endpoint == "" { - conf.Importers.WebAPI.Endpoint = "/sites" - } - if conf.Importers.SiteRegistration.Endpoint == "" { conf.Importers.SiteRegistration.Endpoint = "/sitereg" } diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 8068ddc913..2f646c57d2 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -36,12 +36,6 @@ type Configuration struct { UpdateInterval string `mapstructure:"update_interval"` Importers struct { - WebAPI struct { - Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` - EnabledConnectors []string `mapstructure:"enabled_connectors"` - } `mapstructure:"webapi"` - SiteRegistration struct { Endpoint string `mapstructure:"endpoint"` IsProtected bool `mapstructure:"is_protected"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 4f6dea0883..74ccedbe3c 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,8 +26,6 @@ const ( ) const ( - // ImporterIDWebAPI is the identifier for the WebAPI importer. - ImporterIDWebAPI = "webapi" // ImporterIDSiteRegistration is the identifier for the external site registration importer. ImporterIDSiteRegistration = "sitereg" ) diff --git a/pkg/mentix/exchangers/importers/webapi.go b/pkg/mentix/exchangers/importers/webapi.go deleted file mode 100755 index 2a326d8786..0000000000 --- a/pkg/mentix/exchangers/importers/webapi.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package importers - -import ( - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchangers/importers/webapi" -) - -// WebAPIImporter implements the generic Web API importer. -type WebAPIImporter struct { - BaseRequestImporter -} - -// Activate activates the importer. -func (importer *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { - return err - } - - // Store WebAPI specifics - importer.SetEndpoint(conf.Importers.WebAPI.Endpoint, conf.Importers.WebAPI.IsProtected) - importer.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) - - importer.RegisterActionHandler("register", webapi.HandleRegisterSiteQuery) - importer.RegisterActionHandler("unregister", webapi.HandleUnregisterSiteQuery) - - return nil -} - -// GetID returns the ID of the importer. -func (importer *WebAPIImporter) GetID() string { - return config.ImporterIDWebAPI -} - -// GetName returns the display name of the importer. -func (importer *WebAPIImporter) GetName() string { - return "WebAPI" -} - -func init() { - registerImporter(&WebAPIImporter{}) -} diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go deleted file mode 100755 index 1221e60d77..0000000000 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package webapi - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/utils/network" -) - -func decodeQueryData(data []byte) (*meshdata.MeshData, error) { - site := &meshdata.Site{} - if err := json.Unmarshal(data, site); err != nil { - return nil, err - } - - // Imported sites will be assigned an ID automatically - site.ID = "" - - meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - if err := meshData.Verify(); err != nil { - return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) - } - - meshData.InferMissingData() - return meshData, nil -} - -func handleQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { - meshData, err := decodeQueryData(data) - if err != nil { - return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil - } - meshData.Status = status - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].ID}), nil -} - -// HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") -} - -// HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(_ *meshdata.MeshData, data []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") -} From ddb935a9b3283c90c5d6a962d7e1f11a83c58fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Feb 2021 14:39:16 +0100 Subject: [PATCH 50/59] Use site ID as locations keys --- pkg/mentix/exchangers/exchanger.go | 3 ++- pkg/mentix/exchangers/exporters/siteloc/query.go | 2 +- pkg/mentix/exchangers/exporters/siteloc/types.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/mentix/exchangers/exchanger.go b/pkg/mentix/exchangers/exchanger.go index 7f0fa33971..908b2ba27d 100644 --- a/pkg/mentix/exchangers/exchanger.go +++ b/pkg/mentix/exchangers/exchanger.go @@ -164,7 +164,8 @@ func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) { exchanger.enabledConnectors = connectors } -// MeshData returns the stored mesh data. +// MeshData returns the stored mesh data. The returned data is cloned to prevent accidental data changes. +// Unauthorized sites are also removed if this exchanger doesn't allow them. func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData { return exchanger.cloneMeshData(true) } diff --git a/pkg/mentix/exchangers/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go index db5243a272..ef32d79d26 100755 --- a/pkg/mentix/exchangers/exporters/siteloc/query.go +++ b/pkg/mentix/exchangers/exporters/siteloc/query.go @@ -52,7 +52,7 @@ func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation locations := make([]*SiteLocation, 0, len(meshData.Sites)) for _, site := range meshData.Sites { locations = append(locations, &SiteLocation{ - Site: site.Name, + SiteID: site.ID, FullName: site.FullName, Longitude: site.Longitude, Latitude: site.Latitude, diff --git a/pkg/mentix/exchangers/exporters/siteloc/types.go b/pkg/mentix/exchangers/exporters/siteloc/types.go index 10b2979947..d6dd060ce0 100644 --- a/pkg/mentix/exchangers/exporters/siteloc/types.go +++ b/pkg/mentix/exchangers/exporters/siteloc/types.go @@ -20,7 +20,7 @@ package siteloc // SiteLocation represents the location information of a site. type SiteLocation struct { - Site string `json:"key"` + SiteID string `json:"key"` FullName string `json:"name"` Longitude float32 `json:"longitude"` Latitude float32 `json:"latitude"` From 9a2ff8669a458a2d04ef8065accbdb140785d964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Feb 2021 10:27:20 +0100 Subject: [PATCH 51/59] Cleanup --- internal/http/services/accounts/accounts.go | 40 ++++--- .../http/services/accounts/data/account.go | 1 + internal/http/services/accounts/manager.go | 107 ++++++++---------- .../http/services/accounts/panel/panel.go | 4 +- 4 files changed, 68 insertions(+), 84 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index 2a475d45bc..c8048ca503 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -63,7 +63,8 @@ func (s *svc) Prefix() string { // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { - // This service currently only has one public endpoint (called "register") used for account registration + return []string{"/"} + // This service currently only has one public endpoint used for account registration return []string{config.EndpointCreate} } @@ -214,17 +215,9 @@ func (s *svc) handleList(values url.Values, body []byte) (interface{}, error) { } func (s *svc) handleFind(values url.Values, body []byte) (interface{}, error) { - by := values.Get("by") - value := values.Get("value") - - if len(by) == 0 && len(value) == 0 { - return nil, errors.Errorf("missing search criteria") - } - - // Find the account using the account manager - account, err := s.manager.FindAccount(by, value) + account, err := s.findAccount(values.Get("by"), values.Get("value")) if err != nil { - return nil, errors.Wrap(err, "no user found") + return nil, err } return map[string]interface{}{"account": account}, nil } @@ -272,17 +265,9 @@ func (s *svc) handleRemove(values url.Values, body []byte) (interface{}, error) } func (s *svc) handleIsAuthorized(values url.Values, body []byte) (interface{}, error) { - by := values.Get("by") - value := values.Get("value") - - if len(by) == 0 && len(value) == 0 { - return nil, errors.Errorf("missing search criteria") - } - - // Find the account associated with the given API key - account, err := s.manager.FindAccount(by, value) + account, err := s.findAccount(values.Get("by"), values.Get("value")) if err != nil { - return nil, errors.Wrap(err, "no user with the specified API key found") + return nil, err } return account.Data.Authorized, nil } @@ -325,6 +310,19 @@ func (s *svc) unmarshalRequestData(body []byte) (*data.Account, error) { return account, nil } +func (s *svc) findAccount(by string, value string) (*data.Account, error) { + if len(by) == 0 && len(value) == 0 { + return nil, errors.Errorf("missing search criteria") + } + + // Find the account using the account manager + account, err := s.manager.FindAccount(by, value) + if err != nil { + return nil, errors.Wrap(err, "user not found") + } + return account, nil +} + func parseConfig(m map[string]interface{}) (*config.Configuration, error) { conf := &config.Configuration{} if err := mapstructure.Decode(m, &conf); err != nil { diff --git a/internal/http/services/accounts/data/account.go b/internal/http/services/accounts/data/account.go index c3f11f9435..fe3e03e46b 100644 --- a/internal/http/services/accounts/data/account.go +++ b/internal/http/services/accounts/data/account.go @@ -49,6 +49,7 @@ type AccountData struct { // Accounts holds an array of user accounts. type Accounts = []*Account +// GetSiteID returns the site ID (generated from the API key) for the given account. func (acc *Account) GetSiteID() key.SiteIdentifier { if id, err := key.CalculateSiteID(acc.Data.APIKey, strings.ToLower(acc.Email)); err == nil { return id diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index afb3cec27d..3afdf4ddb8 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -38,8 +38,11 @@ import ( ) const ( - FindByEmail = "email" + // FindByEmail holds the string value of the corresponding search criterium. + FindByEmail = "email" + // FindByAPIKey holds the string value of the corresponding search criterium. FindByAPIKey = "apikey" + // FindBySiteID holds the string value of the corresponding search criterium. FindBySiteID = "siteid" ) @@ -117,42 +120,36 @@ func (mngr *Manager) writeAllAccounts() { } } -func (mngr *Manager) findAccountByEmail(email string) *data.Account { - if email == "" { - return nil +func (mngr *Manager) findAccount(by string, value string) (*data.Account, error) { + if len(value) == 0 { + return nil, errors.Errorf("no search value specified") } - // Perform a case-insensitive search of the given email address - for _, account := range mngr.accounts { - if strings.EqualFold(account.Email, email) { - return account - } - } - return nil -} + var account *data.Account + switch strings.ToLower(by) { + case FindByEmail: + account = mngr.findAccountByPredicate(func(account *data.Account) bool { return strings.EqualFold(account.Email, value) }) -func (mngr *Manager) findAccountByAPIKey(key key.APIKey) *data.Account { - if key == "" { - return nil - } + case FindByAPIKey: + account = mngr.findAccountByPredicate(func(account *data.Account) bool { return account.Data.APIKey == value }) - // Perform a case-sensitive search of the given API key - for _, account := range mngr.accounts { - if account.Data.APIKey == key { - return account - } + case FindBySiteID: + account = mngr.findAccountByPredicate(func(account *data.Account) bool { return account.GetSiteID() == value }) + + default: + return nil, errors.Errorf("invalid search type %v", by) } - return nil -} -func (mngr *Manager) findAccountBySiteID(siteID key.SiteIdentifier) *data.Account { - if siteID == "" { - return nil + if account != nil { + return account, nil } - // Perform a case-sensitive search of the given site ID + return nil, errors.Errorf("no user found matching the specified criteria") +} + +func (mngr *Manager) findAccountByPredicate(predicate func(*data.Account) bool) *data.Account { for _, account := range mngr.accounts { - if account.GetSiteID() == siteID { + if predicate(account) { return account } } @@ -171,7 +168,7 @@ func (mngr *Manager) CreateAccount(accountData *data.Account) error { defer mngr.mutex.Unlock() // Accounts must be unique (identified by their email address) - if mngr.findAccountByEmail(accountData.Email) != nil { + if account, _ := mngr.findAccount(FindByEmail, accountData.Email); account != nil { return errors.Errorf("an account with the specified email address already exists") } @@ -193,9 +190,9 @@ func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) err mngr.mutex.Lock() defer mngr.mutex.Unlock() - account := mngr.findAccountByEmail(accountData.Email) - if account == nil { - return errors.Errorf("no account with the specified email exists") + account, err := mngr.findAccount(FindByEmail, accountData.Email) + if err != nil { + return errors.Wrap(err, "user to update not found") } if err := account.Copy(accountData, copyData); err == nil { @@ -214,28 +211,14 @@ func (mngr *Manager) FindAccount(by string, value string) (*data.Account, error) mngr.mutex.RLock() defer mngr.mutex.RUnlock() - var account *data.Account - switch strings.ToLower(by) { - case FindByEmail: - account = mngr.findAccountByEmail(value) - - case FindByAPIKey: - account = mngr.findAccountByAPIKey(value) - - case FindBySiteID: - account = mngr.findAccountBySiteID(value) - - default: - return nil, errors.Errorf("invalid search type %v", by) + account, err := mngr.findAccount(by, value) + if err != nil { + return nil, err } - if account != nil { - // Clone the account to avoid external data changes - clonedAccount := *account - return &clonedAccount, nil - } - - return nil, errors.Errorf("no user found matching the specified criteria") + // Clone the account to avoid external data changes + clonedAccount := *account + return &clonedAccount, nil } // AuthorizeAccount sets the authorization status of the account identified by the account email; if no such account exists, an error is returned. @@ -243,9 +226,9 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool mngr.mutex.Lock() defer mngr.mutex.Unlock() - account := mngr.findAccountByEmail(accountData.Email) - if account == nil { - return errors.Errorf("no account with the specified email exists") + account, err := mngr.findAccount(FindByEmail, accountData.Email) + if err != nil { + return errors.Wrap(err, "no account with the specified email exists") } authorizedOld := account.Data.Authorized @@ -265,9 +248,9 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8 mngr.mutex.Lock() defer mngr.mutex.Unlock() - account := mngr.findAccountByEmail(accountData.Email) - if account == nil { - return errors.Errorf("no account with the specified email exists") + account, err := mngr.findAccount(FindByEmail, accountData.Email) + if err != nil { + return errors.Wrap(err, "no account with the specified email exists") } if len(account.Data.APIKey) > 0 { @@ -281,10 +264,12 @@ func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8 } // See if the key already exists (super extremely unlikely); if so, generate a new one and try again - if mngr.findAccountByAPIKey(apiKey) == nil { - account.Data.APIKey = apiKey - break + if acc, _ := mngr.findAccount(FindByAPIKey, apiKey); acc != nil { + continue } + + account.Data.APIKey = apiKey + break } mngr.storage.AccountUpdated(account) diff --git a/internal/http/services/accounts/panel/panel.go b/internal/http/services/accounts/panel/panel.go index 9170039950..3eb89a54f9 100644 --- a/internal/http/services/accounts/panel/panel.go +++ b/internal/http/services/accounts/panel/panel.go @@ -62,11 +62,11 @@ func (panel *Panel) Execute(w http.ResponseWriter, accounts *data.Accounts) erro Accounts *data.Accounts } - data := TemplateData{ + tplData := TemplateData{ Accounts: accounts, } - return panel.tpl.Execute(w, data) + return panel.tpl.Execute(w, tplData) } // NewPanel creates a new web interface panel. From c5a861ad11ed294a8fa6adb9e593116de39606b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Feb 2021 10:57:01 +0100 Subject: [PATCH 52/59] Cleanup --- internal/http/services/accounts/accounts.go | 1 - pkg/mentix/accservice/accservice.go | 2 +- .../exchangers/exporters/cs3api/query.go | 8 ++--- .../exchangers/exporters/reqexporter.go | 2 +- .../exchangers/exporters/siteloc/query.go | 8 ++--- .../exchangers/exporters/webapi/query.go | 6 ++-- .../exchangers/importers/reqimporter.go | 2 +- pkg/mentix/exchangers/importers/sitereg.go | 4 +-- pkg/mentix/exchangers/reqexchanger.go | 32 +++++++++++++++---- pkg/mentix/meshdata/site.go | 2 ++ 10 files changed, 44 insertions(+), 23 deletions(-) diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/accounts/accounts.go index c8048ca503..cb7d6365a8 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/accounts/accounts.go @@ -63,7 +63,6 @@ func (s *svc) Prefix() string { // Unprotected returns all endpoints that can be queried without prior authorization. func (s *svc) Unprotected() []string { - return []string{"/"} // This service currently only has one public endpoint used for account registration return []string{config.EndpointCreate} } diff --git a/pkg/mentix/accservice/accservice.go b/pkg/mentix/accservice/accservice.go index 64c5b95061..c77f066853 100644 --- a/pkg/mentix/accservice/accservice.go +++ b/pkg/mentix/accservice/accservice.go @@ -78,13 +78,13 @@ func GetResponseValue(resp *requestResponse, path string) interface{} { if data, ok = data[name].(map[string]interface{}); !ok { break } - } } return nil } +// InitAccountsService initializes the global accounts service. func InitAccountsService(conf *config.Configuration) error { URL, err := url.Parse(conf.AccountsService.URL) if err != nil { diff --git a/pkg/mentix/exchangers/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go index adc923f6e9..21d576ba95 100755 --- a/pkg/mentix/exchangers/exporters/cs3api/query.go +++ b/pkg/mentix/exchangers/exporters/cs3api/query.go @@ -32,20 +32,20 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) { // Convert the mesh data ocmData, err := convertMeshDataToOCMData(meshData) if err != nil { - return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err) } // Marshal the OCM data as JSON data, err := json.MarshalIndent(ocmData, "", "\t") if err != nil { - return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err) } - return nil, http.StatusOK, data, nil + return http.StatusOK, data, nil } func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) { diff --git a/pkg/mentix/exchangers/exporters/reqexporter.go b/pkg/mentix/exchangers/exporters/reqexporter.go index ac03f1ec59..59c15439ce 100644 --- a/pkg/mentix/exchangers/exporters/reqexporter.go +++ b/pkg/mentix/exchangers/exporters/reqexporter.go @@ -47,6 +47,6 @@ func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req } func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (int, []byte, error) { - _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params, conf, log) + _, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params, false, conf, log) return status, data, err } diff --git a/pkg/mentix/exchangers/exporters/siteloc/query.go b/pkg/mentix/exchangers/exporters/siteloc/query.go index ef32d79d26..ba534e7597 100755 --- a/pkg/mentix/exchangers/exporters/siteloc/query.go +++ b/pkg/mentix/exchangers/exporters/siteloc/query.go @@ -31,20 +31,20 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) { // Convert the mesh data locData, err := convertMeshDataToLocationData(meshData) if err != nil { - return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err) } // Marshal the location data as JSON data, err := json.MarshalIndent(locData, "", "\t") if err != nil { - return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err) } - return nil, http.StatusOK, data, nil + return http.StatusOK, data, nil } func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) { diff --git a/pkg/mentix/exchangers/exporters/webapi/query.go b/pkg/mentix/exchangers/exporters/webapi/query.go index 4f596e78a9..aa664bdff5 100755 --- a/pkg/mentix/exchangers/exporters/webapi/query.go +++ b/pkg/mentix/exchangers/exporters/webapi/query.go @@ -31,12 +31,12 @@ import ( ) // HandleDefaultQuery processes a basic query. -func HandleDefaultQuery(meshData *meshdata.MeshData, body []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { +func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) { // Just return the plain, unfiltered data as JSON data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { - return nil, http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) } - return nil, http.StatusOK, data, nil + return http.StatusOK, data, nil } diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index abb5599481..933db84fa5 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -69,5 +69,5 @@ func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values, importer.Locker().RLock() defer importer.Locker().RUnlock() - return importer.HandleAction(importer.MeshData(), data, params, conf, log) + return importer.HandleAction(importer.MeshData(), data, params, true, conf, log) } diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go index 8c1af735c7..34c642527c 100644 --- a/pkg/mentix/exchangers/importers/sitereg.go +++ b/pkg/mentix/exchangers/importers/sitereg.go @@ -41,8 +41,8 @@ func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, l importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) importer.SetAllowUnauthorizedSites(true) - importer.RegisterActionHandler("register", sitereg.HandleRegisterSiteQuery) - importer.RegisterActionHandler("unregister", sitereg.HandleUnregisterSiteQuery) + importer.RegisterExtendedActionHandler("register", sitereg.HandleRegisterSiteQuery) + importer.RegisterExtendedActionHandler("unregister", sitereg.HandleUnregisterSiteQuery) return nil } diff --git a/pkg/mentix/exchangers/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go index cdee85d915..7539b4cfa9 100644 --- a/pkg/mentix/exchangers/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -42,7 +42,8 @@ type RequestExchanger interface { HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) } -type queryCallback func(*meshdata.MeshData, []byte, url.Values, *config.Configuration, *zerolog.Logger) (meshdata.Vector, int, []byte, error) +type queryCallback func(*meshdata.MeshData, url.Values, *config.Configuration, *zerolog.Logger) (int, []byte, error) +type extendedQueryCallback func(*meshdata.MeshData, []byte, url.Values, *config.Configuration, *zerolog.Logger) (meshdata.Vector, int, []byte, error) // BaseRequestExchanger implements basic exporter functionality common to all request exporters. type BaseRequestExchanger struct { @@ -51,7 +52,8 @@ type BaseRequestExchanger struct { endpoint string isProtectedEndpoint bool - actionHandlers map[string]queryCallback + actionHandlers map[string]queryCallback + extendedActionHandlers map[string]extendedQueryCallback } // Endpoint returns the (relative) endpoint of the exchanger. @@ -93,12 +95,30 @@ func (exchanger *BaseRequestExchanger) RegisterActionHandler(action string, call exchanger.actionHandlers[action] = callback } +// RegisterExtendedActionHandler registers a new handler for the specified extended action. +func (exchanger *BaseRequestExchanger) RegisterExtendedActionHandler(action string, callback extendedQueryCallback) { + if exchanger.extendedActionHandlers == nil { + exchanger.extendedActionHandlers = make(map[string]extendedQueryCallback) + } + exchanger.extendedActionHandlers[action] = callback +} + // HandleAction executes the registered handler for the specified action, if any. -func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) { +func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values, isExtended bool, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) { reqAction := params.Get("action") - for action, handler := range exchanger.actionHandlers { - if strings.EqualFold(action, reqAction) { - return handler(meshData, body, params, conf, log) + + if isExtended { + for action, handler := range exchanger.extendedActionHandlers { + if strings.EqualFold(action, reqAction) { + return handler(meshData, body, params, conf, log) + } + } + } else { + for action, handler := range exchanger.actionHandlers { + if strings.EqualFold(action, reqAction) { + status, data, err := handler(meshData, params, conf, log) + return nil, status, data, err + } } } diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 977e341eaa..e48ab6c363 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -130,6 +130,8 @@ func (site *Site) InferMissingData() { } } +// IsAuthorized checks whether the site is authorized. ScienceMesh are always authorized, while for community sites, +// the accounts service is queried. func (site *Site) IsAuthorized() bool { // ScienceMesh sites are always authorized if site.Type == SiteTypeScienceMesh { From 2e724cac73519b8a310ac61337d2678d25d1c00b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Feb 2021 11:08:17 +0100 Subject: [PATCH 53/59] Add/update examples --- examples/accounts/accounts.toml | 19 +++++++++++++++++++ examples/mentix/mentix.toml | 23 +++++++++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) create mode 100644 examples/accounts/accounts.toml diff --git a/examples/accounts/accounts.toml b/examples/accounts/accounts.toml new file mode 100644 index 0000000000..50f8c12701 --- /dev/null +++ b/examples/accounts/accounts.toml @@ -0,0 +1,19 @@ +[http] +address = "0.0.0.0:9600" + +[http.services.accounts] +# All notification emails are sent to this email +notifications_mail = "science.mesh@example.com" + +# Set up the storage driver +[http.services.accounts.storage] +driver = "file" +[http.services.accounts.storage.file] +file = "/var/revad/accounts.json" + +# The SMTP server used for sending emails +[http.services.accounts.smtp] +sender_mail = "science.mesh@example.com" +smtp_server = "mail.example.com" +smtp_port = 25 +disable_auth = true diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 9baf22bbfc..8cdaa11d38 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -1,9 +1,5 @@ -[shared] -jwt_secret = "Ment1x-T0pS3cr3t" - [http] address = "0.0.0.0:9600" -enabled_services = ["mentix"] [http.services.mentix] update_interval = "15m" @@ -25,16 +21,19 @@ endpoint = "/" # If this setting is omitted, all connectors will be used as data sources enabled_connectors = ["gocdb"] -# Enable the WebAPI importer -[http.services.mentix.importers.webapi] +# Enable the site registration importer +[http.services.mentix.importers.sitereg] # For importers, this is obligatory; the connectors will be used as the target for data updates enabled_connectors = ["localfile"] - -# Enable the AdminAPI importer -[http.services.mentix.importers.adminapi] -enabled_connectors = ["localfile"] -# Should never allow access w/o prior authorization -is_protected = true +# If set to true, ScienceMesh sites will be ignored when they try to register +ignore_sm_sites = false + +# Set up the accounts service used to query information about accounts associated with registered sites +[http.services.mentix.accounts] +# Depending on where the service is running, localhost may also be used here +url = "https://sciencemesh.example.com/iop/accounts" +user = "username" +password = "userpass" # Configure the Prometheus Service Discovery: [http.services.mentix.exporters.promsd] From cdf48527402ea18308dd4e1447e7e6a986d906f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Feb 2021 11:46:39 +0100 Subject: [PATCH 54/59] Lint fixes --- pkg/mentix/accservice/accservice.go | 9 +++++---- pkg/mentix/config/config.go | 8 ++++---- pkg/mentix/exchangers/importers/sitereg/types.go | 14 +++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pkg/mentix/accservice/accservice.go b/pkg/mentix/accservice/accservice.go index c77f066853..8d53bbba50 100644 --- a/pkg/mentix/accservice/accservice.go +++ b/pkg/mentix/accservice/accservice.go @@ -31,7 +31,8 @@ import ( "github.com/cs3org/reva/pkg/mentix/utils/network" ) -type requestResponse struct { +// RequestResponse holds the response of an accounts service query. +type RequestResponse struct { Success bool Error string Data interface{} @@ -46,7 +47,7 @@ type accountsServiceSettings struct { var settings accountsServiceSettings // Query performs an account service query. -func Query(endpoint string, params network.URLParams) (*requestResponse, error) { +func Query(endpoint string, params network.URLParams) (*RequestResponse, error) { fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", settings.URL.Scheme, settings.URL.Host), path.Join(settings.URL.Path, endpoint), params) if err != nil { return nil, errors.Wrap(err, "error while building the service accounts query URL") @@ -57,7 +58,7 @@ func Query(endpoint string, params network.URLParams) (*requestResponse, error) return nil, errors.Wrap(err, "unable to query the service accounts endpoint") } - resp := &requestResponse{} + resp := &RequestResponse{} if err := json.Unmarshal(data, resp); err != nil { return nil, errors.Wrap(err, "unable to unmarshal response data") } @@ -65,7 +66,7 @@ func Query(endpoint string, params network.URLParams) (*requestResponse, error) } // GetResponseValue gets a value from an account service query using a dotted path notation. -func GetResponseValue(resp *requestResponse, path string) interface{} { +func GetResponseValue(resp *RequestResponse, path string) interface{} { if data, ok := resp.Data.(map[string]interface{}); ok { tokens := strings.Split(path, ".") for i, name := range tokens { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 2f646c57d2..75d44780b2 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -38,8 +38,8 @@ type Configuration struct { Importers struct { SiteRegistration struct { Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` + IsProtected bool `mapstructure:"is_protected"` IgnoreScienceMeshSites bool `mapstructure:"ignore_sm_sites"` } `mapstructure:"sitereg"` } `mapstructure:"importers"` @@ -47,20 +47,20 @@ type Configuration struct { Exporters struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` + IsProtected bool `mapstructure:"is_protected"` } `mapstructure:"webapi"` CS3API struct { Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` + IsProtected bool `mapstructure:"is_protected"` } `mapstructure:"cs3api"` SiteLocations struct { Endpoint string `mapstructure:"endpoint"` - IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` + IsProtected bool `mapstructure:"is_protected"` } `mapstructure:"siteloc"` PrometheusSD struct { diff --git a/pkg/mentix/exchangers/importers/sitereg/types.go b/pkg/mentix/exchangers/importers/sitereg/types.go index e64fe8e9f0..b10bcc39dc 100644 --- a/pkg/mentix/exchangers/importers/sitereg/types.go +++ b/pkg/mentix/exchangers/importers/sitereg/types.go @@ -83,21 +83,21 @@ func (siteData *siteRegistrationData) Verify() error { // ToMeshDataSite converts the stored data into a meshdata site object, filling out as much data as possible. func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, siteType meshdata.SiteType, email string) (*meshdata.Site, error) { - siteUrl, err := url.Parse(siteData.URL) + siteURL, err := url.Parse(siteData.URL) if err != nil { return nil, errors.Wrap(err, "invalid site URL") } // Create the Reva service entry revaHost := siteData.Reva.Host - revaUrl := siteData.Reva.URL + revaURL := siteData.Reva.URL if len(revaHost) == 0 { // Infer host from URL - URL, _ := url.Parse(revaUrl) + URL, _ := url.Parse(revaURL) revaHost = network.ExtractDomainFromURL(URL, true) - } else if len(revaUrl) == 0 { // Infer URL from host + } else if len(revaURL) == 0 { // Infer URL from host URL, _ := network.GenerateURL(revaHost, "", network.URLParams{}) - revaUrl = URL.String() + revaURL = URL.String() } properties := make(map[string]string, 1) @@ -110,7 +110,7 @@ func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, Description: "Reva Daemon", }, Name: revaHost + " - REVAD", - URL: revaUrl, + URL: revaURL, IsMonitored: true, Properties: properties, }, @@ -125,7 +125,7 @@ func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, Name: siteData.Name, FullName: siteData.Name, Organization: "", - Domain: network.ExtractDomainFromURL(siteUrl, true), + Domain: network.ExtractDomainFromURL(siteURL, true), Homepage: siteData.URL, Email: email, Description: siteData.Name + " @ " + siteData.URL, From 6c0e74065d57a3332b20585d0c88c62ac635422b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Feb 2021 11:55:52 +0100 Subject: [PATCH 55/59] Update documentation --- .../config/http/services/accounts/_index.md | 95 +++++++++++++++++++ .../config/http/services/mentix/_index.md | 35 ++++++- .../http/services/mentix/adminapi/_index.md | 46 --------- .../http/services/mentix/sitereg/_index.md | 37 ++++++++ .../http/services/mentix/webapi/_index.md | 42 +------- 5 files changed, 163 insertions(+), 92 deletions(-) create mode 100644 docs/content/en/docs/config/http/services/accounts/_index.md delete mode 100644 docs/content/en/docs/config/http/services/mentix/adminapi/_index.md create mode 100644 docs/content/en/docs/config/http/services/mentix/sitereg/_index.md diff --git a/docs/content/en/docs/config/http/services/accounts/_index.md b/docs/content/en/docs/config/http/services/accounts/_index.md new file mode 100644 index 0000000000..3f04a79a0c --- /dev/null +++ b/docs/content/en/docs/config/http/services/accounts/_index.md @@ -0,0 +1,95 @@ +--- +title: "accounts" +linkTitle: "accounts" +weight: 10 +description: > + Configuration for the Accounts service +--- + +{{% pageinfo %}} +The accounts service is used to store and manage user accounts. +{{% /pageinfo %}} + +## General settings +{{% dir name="prefix" type="string" default="accounts" %}} +The relative root path of all exposed HTTP endpoints of the service. +{{< highlight toml >}} +[http.services.accounts] +prefix = "/accounts" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="notifications_mail" type="string" default="" %}} +An email address where all notifications are sent to. +{{< highlight toml >}} +[http.services.accounts] +notifications_mail = "notify@example.com" +{{< /highlight >}} +{{% /dir %}} + +## SMTP settings +{{% dir name="sender_mail" type="string" default="" %}} +An email address from which all emails are sent. +{{< highlight toml >}} +[http.services.accounts.smtp] +sender_mail = "notify@example.com" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sender_login" type="string" default="" %}} +The login name. +{{< highlight toml >}} +[http.services.accounts.smtp] +sender_login = "hans" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sender_password" type="string" default="" %}} +The password for the login. +{{< highlight toml >}} +[http.services.accounts.smtp] +password = "secret" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="smtp_server" type="string" default="" %}} +The SMTP server to use. +{{< highlight toml >}} +[http.services.accounts.smtp] +smtp_server = "smtp.example.com" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="smtp_port" type="int" default="25" %}} +The SMTP server port to use. +{{< highlight toml >}} +[http.services.accounts.smtp] +smtp_port = 25 +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="disable_auth" type="bool" default="false" %}} +Whether to disable authentication. +{{< highlight toml >}} +[http.services.accounts.smtp] +disable_auth = true +{{< /highlight >}} +{{% /dir %}} + +## Storage settings +{{% dir name="driver" type="string" default="file" %}} +The storage driver to use; currently, only `file` is supported. +{{< highlight toml >}} +[http.services.accounts.storage] +driver = "file" +{{< /highlight >}} +{{% /dir %}} + +### Storage settings - File driver +{{% dir name="file" type="string" default="" %}} +The file location. +{{< highlight toml >}} +[http.services.accounts.storage.file] +file = "/var/reva/accounts.json" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 8e12bc5be7..9fa556bf69 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -10,6 +10,7 @@ description: > Mentix (_**Me**sh E**nti**ty E**x**changer_) is a service to read and write mesh topology data to and from one or more sources (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. {{% /pageinfo %}} +## General settings {{% dir name="prefix" type="string" default="mentix" %}} The relative root path of all exposed HTTP endpoints of Mentix. {{< highlight toml >}} @@ -42,11 +43,8 @@ Mentix can import mesh data from various sources and write it to one or more tar __Supported importers:__ -- **webapi** -Mentix can import mesh data via an HTTP endpoint using the `webapi` importer. Data can be sent to the configured relative endpoint (see [here](webapi)). - -- **adminapi** - Some aspects of Mentix can be administered through an HTTP endpoint using the `adminapi` importer. Queries can be sent to the configured relative endpoint (see [here](adminapi)). +- **sitereg** +Mentix can import new sites via an HTTP endpoint using the `sitereg` importer. Data can be sent to the configured relative endpoint (see [here](sitereg)). ## Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. @@ -65,3 +63,30 @@ Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data c - files: - '/usr/share/prom/sciencemesh_services.json' ``` + +## Accounts service +Mentix uses the Reva accounts service to query information about user accounts. The following settings under the `accounts` group must be configured properly: + +{{% dir name="url" type="string" default="" %}} +The URL of the accounts service. +{{< highlight toml >}} +[http.services.mentix.accounts] +url = "https://example.com/accounts" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="user" type="string" default="" %}} +The user name to use for basic HTTP authentication. +{{< highlight toml >}} +[http.services.mentix.accounts] +user = "hans" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="password" type="string" default="" %}} +The user password to use for basic HTTP authentication. +{{< highlight toml >}} +[http.services.mentix.accounts] +password = "secret" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md b/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md deleted file mode 100644 index 55f29684eb..0000000000 --- a/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: "adminapi" -linkTitle: "adminapi" -weight: 10 -description: > - Configuration for the AdminAPI of the Mentix service ---- - -{{% pageinfo %}} -The AdminAPI of Mentix is a special importer that can be used to administer certain aspects of Mentix. -{{% /pageinfo %}} - -The AdminAPI importer receives instructions/queries through a `POST` request. - -The importer supports one action that must be passed in the URL: -``` -https://sciencemesh.example.com/mentix/admin/?action= -``` -Currently, the following actions are supported: -- `authorize`: Authorizes or unauthorizes a site - -For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. - -{{% dir name="endpoint" type="string" default="/admin" %}} -The endpoint where the mesh data can be sent to. -{{< highlight toml >}} -[http.services.mentix.importers.adminapi] -endpoint = "/data" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="is_protected" type="bool" default="false" %}} -Whether the endpoint requires authentication. -{{< highlight toml >}} -[http.services.mentix.importers.adminapi] -is_protected = true -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="enabled_connectors" type="[]string" default="" %}} -A list of all enabled connectors for the importer. Must always be provided. -{{< highlight toml >}} -[http.services.mentix.importers.adminapi] -enabled_connectors = ["localfile"] -{{< /highlight >}} -{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md b/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md new file mode 100644 index 0000000000..3591732b9e --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md @@ -0,0 +1,37 @@ +--- +title: "sitereg" +linkTitle: "sitereg" +weight: 10 +description: > + Configuration for site registration service +--- + +{{% pageinfo %}} +The site registration service is used to register new and unregister existing sites. +{{% /pageinfo %}} + +The site registration service is used to register new and unregister existing sites. + +{{% dir name="endpoint" type="string" default="/sitereg" %}} +The endpoint of the service. +{{< highlight toml >}} +[http.services.mentix.importers.sitereg] +endpoint = "/reg" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="" %}} +A list of all enabled connectors for the importer. +{{< highlight toml >}} +[http.services.mentix.importers.sitereg] +enabled_connectors = ["localfile"] +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="ignore_sm_sites" type="bool" default="false" %}} +If set to true, registrations from ScienceMesh sites will be ignored. +{{< highlight toml >}} +[http.services.mentix.importers.sitereg] +ignore_sm_sites = true +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index 03e4c028bc..f3ebd77bfc 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -7,49 +7,9 @@ description: > --- {{% pageinfo %}} -The WebAPI of Mentix supports both importing and exporting of mesh data via an HTTP endpoint. Both the im- and exporter are configured separately. +The WebAPI of Mentix supports of mesh data via an HTTP endpoint. {{% /pageinfo %}} -## Importer - -The WebAPI importer receives a single _plain_ Mentix site through an HTTP `POST` request; service types are currently not supported. - -The importer supports two actions that must be passed in the URL: -``` -https://sciencemesh.example.com/mentix/webapi/?action= -``` -Currently, the following actions are supported: -- `register`: Registers a new site -- `unregister`: Unregisters an existing site - -For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. - -{{% dir name="endpoint" type="string" default="/sites" %}} -The endpoint where the mesh data can be sent to. -{{< highlight toml >}} -[http.services.mentix.importers.webapi] -endpoint = "/data" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="is_protected" type="bool" default="false" %}} -Whether the endpoint requires authentication. -{{< highlight toml >}} -[http.services.mentix.importers.webapi] -is_protected = true -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="enabled_connectors" type="[]string" default="" %}} -A list of all enabled connectors for the importer. Must always be provided. -{{< highlight toml >}} -[http.services.mentix.importers.webapi] -enabled_connectors = ["localfile"] -{{< /highlight >}} -{{% /dir %}} - -## Exporter - The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. {{% dir name="endpoint" type="string" default="/sites" %}} From 79f798c9a61cbb73aa81d4542b9438e0b9cb110e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 1 Mar 2021 12:30:51 +0100 Subject: [PATCH 56/59] Add changelog --- changelog/unreleased/accounts-svc.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/accounts-svc.md diff --git a/changelog/unreleased/accounts-svc.md b/changelog/unreleased/accounts-svc.md new file mode 100644 index 0000000000..72b47561d1 --- /dev/null +++ b/changelog/unreleased/accounts-svc.md @@ -0,0 +1,7 @@ +Enhancement: User Accounts service for API keys + +This update adds a new service to Reva that handles user accounts creation and management. Registered users can be assigned an API key through a simple web interface which is also part of this service. This API key can then be used to identify a user and his/her associated (vendor or partner) site. + +Furthermore, Mentix was extended to make use of this new service. This way, all sites now have a stable and unique site ID that not only avoids ID collisions but also introduces a new layer of security (i.e., sites can only be modified or removed using the correct API key). + +https://github.com/cs3org/reva/pull/1506 From f2900ad17b0f8bed95a73c31187b1a89d930250e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 1 Mar 2021 12:38:11 +0100 Subject: [PATCH 57/59] Hound fixes --- internal/http/services/accounts/config/endpoints.go | 2 +- internal/http/services/accounts/data/filestorage.go | 4 ++-- internal/http/services/accounts/data/storage.go | 4 ++-- internal/http/services/accounts/manager.go | 3 +++ internal/http/services/accounts/panel/panel.go | 1 + pkg/mentix/utils/network/network.go | 1 + 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index 2332d3a42e..9cd5fc41e7 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -24,7 +24,7 @@ const ( // EndpointGenerateAPIKey is the endpoint path of the API key generator. EndpointGenerateAPIKey = "/generate-api-key" - // EndpointGenerateAPIKey is the endpoint path for API key verification. + // EndpointVerifyAPIKey is the endpoint path for API key verification. EndpointVerifyAPIKey = "/verify-api-key" // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. EndpointAssignAPIKey = "/assign-api-key" diff --git a/internal/http/services/accounts/data/filestorage.go b/internal/http/services/accounts/data/filestorage.go index fb9ec17c80..4705c215c6 100644 --- a/internal/http/services/accounts/data/filestorage.go +++ b/internal/http/services/accounts/data/filestorage.go @@ -96,12 +96,12 @@ func (storage *FileStorage) AccountAdded(account *Account) { // Simply skip this action; all data is saved solely in WriteAll } -// AccountAdded is called when an account has been updated. +// AccountUpdated is called when an account has been updated. func (storage *FileStorage) AccountUpdated(account *Account) { // Simply skip this action; all data is saved solely in WriteAll } -// AccountAdded is called when an account has been removed. +// AccountRemoved is called when an account has been removed. func (storage *FileStorage) AccountRemoved(account *Account) { // Simply skip this action; all data is saved solely in WriteAll } diff --git a/internal/http/services/accounts/data/storage.go b/internal/http/services/accounts/data/storage.go index 58269c72ba..221e7bcb32 100644 --- a/internal/http/services/accounts/data/storage.go +++ b/internal/http/services/accounts/data/storage.go @@ -27,8 +27,8 @@ type Storage interface { // AccountAdded is called when an account has been added. AccountAdded(account *Account) - // AccountAdded is called when an account has been updated. + // AccountUpdated is called when an account has been updated. AccountUpdated(account *Account) - // AccountAdded is called when an account has been removed. + // AccountRemoved is called when an account has been removed. AccountRemoved(account *Account) } diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/accounts/manager.go index 3afdf4ddb8..849e03630c 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/accounts/manager.go @@ -156,6 +156,7 @@ func (mngr *Manager) findAccountByPredicate(predicate func(*data.Account) bool) return nil } +// ShowPanel writes the panel HTTP output directly to the response writer. func (mngr *Manager) ShowPanel(w http.ResponseWriter) error { // The panel only shows the stored accounts and offers actions through links, so let it use cloned data accounts := mngr.CloneAccounts() @@ -207,6 +208,7 @@ func (mngr *Manager) UpdateAccount(accountData *data.Account, copyData bool) err return nil } +// FindAccount is used to find an account by various criteria. func (mngr *Manager) FindAccount(by string, value string) (*data.Account, error) { mngr.mutex.RLock() defer mngr.mutex.RUnlock() @@ -244,6 +246,7 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool return nil } +// AssignAPIKeyToAccount is used to assign a new API key to the account identified by the account email; if no such account exists, an error is returned. func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8) error { mngr.mutex.Lock() defer mngr.mutex.Unlock() diff --git a/internal/http/services/accounts/panel/panel.go b/internal/http/services/accounts/panel/panel.go index 3eb89a54f9..87d1b4d35c 100644 --- a/internal/http/services/accounts/panel/panel.go +++ b/internal/http/services/accounts/panel/panel.go @@ -57,6 +57,7 @@ func (panel *Panel) initialize(conf *config.Configuration, log *zerolog.Logger) return nil } +// Execute generates the HTTP output of the panel and writes it to the response writer. func (panel *Panel) Execute(w http.ResponseWriter, accounts *data.Accounts) error { type TemplateData struct { Accounts *data.Accounts diff --git a/pkg/mentix/utils/network/network.go b/pkg/mentix/utils/network/network.go index 9704b8b578..da6fa65bec 100644 --- a/pkg/mentix/utils/network/network.go +++ b/pkg/mentix/utils/network/network.go @@ -35,6 +35,7 @@ type URLParams map[string]string // ResponseParams holds parameters of an HTTP response. type ResponseParams map[string]interface{} +// BasicAuth holds user credentials for basic HTTP authentication. type BasicAuth struct { User string Password string From 3c8f8c6db1961c56307c46874691e80ac2f7e286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 1 Mar 2021 12:39:15 +0100 Subject: [PATCH 58/59] Hound fixes --- internal/http/services/accounts/config/endpoints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/accounts/config/endpoints.go index 9cd5fc41e7..80541bf803 100644 --- a/internal/http/services/accounts/config/endpoints.go +++ b/internal/http/services/accounts/config/endpoints.go @@ -26,7 +26,7 @@ const ( EndpointGenerateAPIKey = "/generate-api-key" // EndpointVerifyAPIKey is the endpoint path for API key verification. EndpointVerifyAPIKey = "/verify-api-key" - // EndpointAssignsAPIKey is the endpoint path used for assigning an API key to an account. + // EndpointAssignAPIKey is the endpoint path used for assigning an API key to an account. EndpointAssignAPIKey = "/assign-api-key" // EndpointList is the endpoint path for listing all stored accounts. From 064cf0ac6799c2458d0a3ccf6a8e3436f26dcdd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 1 Mar 2021 14:13:45 +0100 Subject: [PATCH 59/59] Rename 'accounts' to 'siteacc' --- .../{accounts-svc.md => siteaccs-svc.md} | 4 +-- .../config/http/services/mentix/_index.md | 6 ++-- .../services/{accounts => siteacc}/_index.md | 30 +++++++++---------- .../accounts.toml => siteacc/siteacc.toml} | 8 ++--- internal/http/services/loader/loader.go | 2 +- .../{accounts => siteacc}/config/config.go | 0 .../{accounts => siteacc}/config/endpoints.go | 0 .../{accounts => siteacc}/data/account.go | 8 ++--- .../{accounts => siteacc}/data/filestorage.go | 2 +- .../{accounts => siteacc}/data/storage.go | 0 .../{accounts => siteacc}/email/email.go | 4 +-- .../{accounts => siteacc}/email/template.go | 0 .../services/{accounts => siteacc}/manager.go | 16 +++++----- .../{accounts => siteacc}/panel/panel.go | 4 +-- .../{accounts => siteacc}/panel/template.go | 0 .../accounts.go => siteacc/siteacc.go} | 12 ++++---- .../exchangers/importers/sitereg/query.go | 2 +- pkg/mentix/key/apikey.go | 12 ++++---- 18 files changed, 55 insertions(+), 55 deletions(-) rename changelog/unreleased/{accounts-svc.md => siteaccs-svc.md} (73%) rename docs/content/en/docs/config/http/services/{accounts => siteacc}/_index.md (80%) rename examples/{accounts/accounts.toml => siteacc/siteacc.toml} (74%) rename internal/http/services/{accounts => siteacc}/config/config.go (100%) rename internal/http/services/{accounts => siteacc}/config/endpoints.go (100%) rename internal/http/services/{accounts => siteacc}/data/account.go (93%) rename internal/http/services/{accounts => siteacc}/data/filestorage.go (98%) rename internal/http/services/{accounts => siteacc}/data/storage.go (100%) rename internal/http/services/{accounts => siteacc}/email/email.go (95%) rename internal/http/services/{accounts => siteacc}/email/template.go (100%) rename internal/http/services/{accounts => siteacc}/manager.go (95%) rename internal/http/services/{accounts => siteacc}/panel/panel.go (94%) rename internal/http/services/{accounts => siteacc}/panel/template.go (100%) rename internal/http/services/{accounts/accounts.go => siteacc/siteacc.go} (97%) diff --git a/changelog/unreleased/accounts-svc.md b/changelog/unreleased/siteaccs-svc.md similarity index 73% rename from changelog/unreleased/accounts-svc.md rename to changelog/unreleased/siteaccs-svc.md index 72b47561d1..45e98ed2d7 100644 --- a/changelog/unreleased/accounts-svc.md +++ b/changelog/unreleased/siteaccs-svc.md @@ -1,6 +1,6 @@ -Enhancement: User Accounts service for API keys +Enhancement: Site Accounts service for API keys -This update adds a new service to Reva that handles user accounts creation and management. Registered users can be assigned an API key through a simple web interface which is also part of this service. This API key can then be used to identify a user and his/her associated (vendor or partner) site. +This update adds a new service to Reva that handles site accounts creation and management. Registered sites can be assigned an API key through a simple web interface which is also part of this service. This API key can then be used to identify a user and his/her associated (vendor or partner) site. Furthermore, Mentix was extended to make use of this new service. This way, all sites now have a stable and unique site ID that not only avoids ID collisions but also introduces a new layer of security (i.e., sites can only be modified or removed using the correct API key). diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 9fa556bf69..62a614e6e3 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -64,11 +64,11 @@ Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data c - '/usr/share/prom/sciencemesh_services.json' ``` -## Accounts service -Mentix uses the Reva accounts service to query information about user accounts. The following settings under the `accounts` group must be configured properly: +## Site Accounts service +Mentix uses the Reva site accounts service to query information about site accounts. The following settings must be configured properly: {{% dir name="url" type="string" default="" %}} -The URL of the accounts service. +The URL of the site accounts service. {{< highlight toml >}} [http.services.mentix.accounts] url = "https://example.com/accounts" diff --git a/docs/content/en/docs/config/http/services/accounts/_index.md b/docs/content/en/docs/config/http/services/siteacc/_index.md similarity index 80% rename from docs/content/en/docs/config/http/services/accounts/_index.md rename to docs/content/en/docs/config/http/services/siteacc/_index.md index 3f04a79a0c..425d3f0c7c 100644 --- a/docs/content/en/docs/config/http/services/accounts/_index.md +++ b/docs/content/en/docs/config/http/services/siteacc/_index.md @@ -1,28 +1,28 @@ --- -title: "accounts" -linkTitle: "accounts" +title: "siteacc" +linkTitle: "siteacc" weight: 10 description: > - Configuration for the Accounts service + Configuration for the Site Accounts service --- {{% pageinfo %}} -The accounts service is used to store and manage user accounts. +The site accounts service is used to store and manage site accounts. {{% /pageinfo %}} ## General settings {{% dir name="prefix" type="string" default="accounts" %}} The relative root path of all exposed HTTP endpoints of the service. {{< highlight toml >}} -[http.services.accounts] -prefix = "/accounts" +[http.services.siteacc] +prefix = "/siteacc" {{< /highlight >}} {{% /dir %}} {{% dir name="notifications_mail" type="string" default="" %}} An email address where all notifications are sent to. {{< highlight toml >}} -[http.services.accounts] +[http.services.siteacc] notifications_mail = "notify@example.com" {{< /highlight >}} {{% /dir %}} @@ -31,7 +31,7 @@ notifications_mail = "notify@example.com" {{% dir name="sender_mail" type="string" default="" %}} An email address from which all emails are sent. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] sender_mail = "notify@example.com" {{< /highlight >}} {{% /dir %}} @@ -39,7 +39,7 @@ sender_mail = "notify@example.com" {{% dir name="sender_login" type="string" default="" %}} The login name. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] sender_login = "hans" {{< /highlight >}} {{% /dir %}} @@ -47,7 +47,7 @@ sender_login = "hans" {{% dir name="sender_password" type="string" default="" %}} The password for the login. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] password = "secret" {{< /highlight >}} {{% /dir %}} @@ -55,7 +55,7 @@ password = "secret" {{% dir name="smtp_server" type="string" default="" %}} The SMTP server to use. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] smtp_server = "smtp.example.com" {{< /highlight >}} {{% /dir %}} @@ -63,7 +63,7 @@ smtp_server = "smtp.example.com" {{% dir name="smtp_port" type="int" default="25" %}} The SMTP server port to use. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] smtp_port = 25 {{< /highlight >}} {{% /dir %}} @@ -71,7 +71,7 @@ smtp_port = 25 {{% dir name="disable_auth" type="bool" default="false" %}} Whether to disable authentication. {{< highlight toml >}} -[http.services.accounts.smtp] +[http.services.siteacc.smtp] disable_auth = true {{< /highlight >}} {{% /dir %}} @@ -80,7 +80,7 @@ disable_auth = true {{% dir name="driver" type="string" default="file" %}} The storage driver to use; currently, only `file` is supported. {{< highlight toml >}} -[http.services.accounts.storage] +[http.services.siteacc.storage] driver = "file" {{< /highlight >}} {{% /dir %}} @@ -89,7 +89,7 @@ driver = "file" {{% dir name="file" type="string" default="" %}} The file location. {{< highlight toml >}} -[http.services.accounts.storage.file] +[http.services.siteacc.storage.file] file = "/var/reva/accounts.json" {{< /highlight >}} {{% /dir %}} diff --git a/examples/accounts/accounts.toml b/examples/siteacc/siteacc.toml similarity index 74% rename from examples/accounts/accounts.toml rename to examples/siteacc/siteacc.toml index 50f8c12701..a03c393065 100644 --- a/examples/accounts/accounts.toml +++ b/examples/siteacc/siteacc.toml @@ -1,18 +1,18 @@ [http] address = "0.0.0.0:9600" -[http.services.accounts] +[http.services.siteacc] # All notification emails are sent to this email notifications_mail = "science.mesh@example.com" # Set up the storage driver -[http.services.accounts.storage] +[http.services.siteacc.storage] driver = "file" -[http.services.accounts.storage.file] +[http.services.siteacc.storage.file] file = "/var/revad/accounts.json" # The SMTP server used for sending emails -[http.services.accounts.smtp] +[http.services.siteacc.smtp] sender_mail = "science.mesh@example.com" smtp_server = "mail.example.com" smtp_port = 25 diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 4a5c12ab98..91f59d051e 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -20,7 +20,6 @@ package loader import ( // Load core HTTP services - _ "github.com/cs3org/reva/internal/http/services/accounts" _ "github.com/cs3org/reva/internal/http/services/datagateway" _ "github.com/cs3org/reva/internal/http/services/dataprovider" _ "github.com/cs3org/reva/internal/http/services/helloworld" @@ -32,6 +31,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocs" _ "github.com/cs3org/reva/internal/http/services/prometheus" + _ "github.com/cs3org/reva/internal/http/services/siteacc" _ "github.com/cs3org/reva/internal/http/services/sysinfo" _ "github.com/cs3org/reva/internal/http/services/wellknown" // Add your own service here diff --git a/internal/http/services/accounts/config/config.go b/internal/http/services/siteacc/config/config.go similarity index 100% rename from internal/http/services/accounts/config/config.go rename to internal/http/services/siteacc/config/config.go diff --git a/internal/http/services/accounts/config/endpoints.go b/internal/http/services/siteacc/config/endpoints.go similarity index 100% rename from internal/http/services/accounts/config/endpoints.go rename to internal/http/services/siteacc/config/endpoints.go diff --git a/internal/http/services/accounts/data/account.go b/internal/http/services/siteacc/data/account.go similarity index 93% rename from internal/http/services/accounts/data/account.go rename to internal/http/services/siteacc/data/account.go index fe3e03e46b..dce4ea7b1a 100644 --- a/internal/http/services/accounts/data/account.go +++ b/internal/http/services/siteacc/data/account.go @@ -28,7 +28,7 @@ import ( "github.com/cs3org/reva/pkg/utils" ) -// Account represents a single user account. +// Account represents a single site account. type Account struct { Email string `json:"email"` FirstName string `json:"firstName"` @@ -40,13 +40,13 @@ type Account struct { Data AccountData `json:"data"` } -// AccountData holds additional data for a user account. +// AccountData holds additional data for a site account. type AccountData struct { APIKey key.APIKey `json:"apiKey"` Authorized bool `json:"authorized"` } -// Accounts holds an array of user accounts. +// Accounts holds an array of site accounts. type Accounts = []*Account // GetSiteID returns the site ID (generated from the API key) for the given account. @@ -89,7 +89,7 @@ func (acc *Account) verify() error { return nil } -// NewAccount creates a new user account. +// NewAccount creates a new site account. func NewAccount(email string, firstName, lastName string) (*Account, error) { t := time.Now() diff --git a/internal/http/services/accounts/data/filestorage.go b/internal/http/services/siteacc/data/filestorage.go similarity index 98% rename from internal/http/services/accounts/data/filestorage.go rename to internal/http/services/siteacc/data/filestorage.go index 4705c215c6..22efb99b3e 100644 --- a/internal/http/services/accounts/data/filestorage.go +++ b/internal/http/services/siteacc/data/filestorage.go @@ -27,7 +27,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/cs3org/reva/internal/http/services/accounts/config" + "github.com/cs3org/reva/internal/http/services/siteacc/config" ) // FileStorage implements a filePath-based storage. diff --git a/internal/http/services/accounts/data/storage.go b/internal/http/services/siteacc/data/storage.go similarity index 100% rename from internal/http/services/accounts/data/storage.go rename to internal/http/services/siteacc/data/storage.go diff --git a/internal/http/services/accounts/email/email.go b/internal/http/services/siteacc/email/email.go similarity index 95% rename from internal/http/services/accounts/email/email.go rename to internal/http/services/siteacc/email/email.go index 009048bc23..72e15f3c66 100644 --- a/internal/http/services/accounts/email/email.go +++ b/internal/http/services/siteacc/email/email.go @@ -24,13 +24,13 @@ import ( "github.com/pkg/errors" - "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/internal/http/services/siteacc/data" "github.com/cs3org/reva/pkg/smtpclient" ) // SendAccountCreated sends an email about account creation. func SendAccountCreated(account *data.Account, recipients []string, smtp *smtpclient.SMTPCredentials) error { - return send(recipients, "ScienceMesh: User account created", accountCreatedTemplate, account, smtp) + return send(recipients, "ScienceMesh: Site account created", accountCreatedTemplate, account, smtp) } // SendAPIKeyAssigned sends an email about API key assignment. diff --git a/internal/http/services/accounts/email/template.go b/internal/http/services/siteacc/email/template.go similarity index 100% rename from internal/http/services/accounts/email/template.go rename to internal/http/services/siteacc/email/template.go diff --git a/internal/http/services/accounts/manager.go b/internal/http/services/siteacc/manager.go similarity index 95% rename from internal/http/services/accounts/manager.go rename to internal/http/services/siteacc/manager.go index 849e03630c..f625dd96a3 100644 --- a/internal/http/services/accounts/manager.go +++ b/internal/http/services/siteacc/manager.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package accounts +package siteacc import ( "bytes" @@ -29,10 +29,10 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/cs3org/reva/internal/http/services/accounts/config" - "github.com/cs3org/reva/internal/http/services/accounts/data" - "github.com/cs3org/reva/internal/http/services/accounts/email" - "github.com/cs3org/reva/internal/http/services/accounts/panel" + "github.com/cs3org/reva/internal/http/services/siteacc/config" + "github.com/cs3org/reva/internal/http/services/siteacc/data" + "github.com/cs3org/reva/internal/http/services/siteacc/email" + "github.com/cs3org/reva/internal/http/services/siteacc/panel" "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/smtpclient" ) @@ -46,7 +46,7 @@ const ( FindBySiteID = "siteid" ) -// Manager is responsible for all user account related tasks. +// Manager is responsible for all site account related tasks. type Manager struct { conf *config.Configuration log *zerolog.Logger @@ -73,7 +73,7 @@ func (mngr *Manager) initialize(conf *config.Configuration, log *zerolog.Logger) mngr.accounts = make(data.Accounts, 0, 32) // Reserve some space for accounts - // Create the user accounts storage and read all stored data + // Create the site accounts storage and read all stored data if storage, err := mngr.createStorage(conf.Storage.Driver); err == nil { mngr.storage = storage mngr.readAllAccounts() @@ -247,7 +247,7 @@ func (mngr *Manager) AuthorizeAccount(accountData *data.Account, authorized bool } // AssignAPIKeyToAccount is used to assign a new API key to the account identified by the account email; if no such account exists, an error is returned. -func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int8) error { +func (mngr *Manager) AssignAPIKeyToAccount(accountData *data.Account, flags int) error { mngr.mutex.Lock() defer mngr.mutex.Unlock() diff --git a/internal/http/services/accounts/panel/panel.go b/internal/http/services/siteacc/panel/panel.go similarity index 94% rename from internal/http/services/accounts/panel/panel.go rename to internal/http/services/siteacc/panel/panel.go index 87d1b4d35c..ce8d6cd916 100644 --- a/internal/http/services/accounts/panel/panel.go +++ b/internal/http/services/siteacc/panel/panel.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/cs3org/reva/internal/http/services/accounts/config" - "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/internal/http/services/siteacc/config" + "github.com/cs3org/reva/internal/http/services/siteacc/data" ) // Panel represents the web interface panel of the accounts service. diff --git a/internal/http/services/accounts/panel/template.go b/internal/http/services/siteacc/panel/template.go similarity index 100% rename from internal/http/services/accounts/panel/template.go rename to internal/http/services/siteacc/panel/template.go diff --git a/internal/http/services/accounts/accounts.go b/internal/http/services/siteacc/siteacc.go similarity index 97% rename from internal/http/services/accounts/accounts.go rename to internal/http/services/siteacc/siteacc.go index cb7d6365a8..1d49e73570 100644 --- a/internal/http/services/accounts/accounts.go +++ b/internal/http/services/siteacc/siteacc.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package accounts +package siteacc import ( "encoding/json" @@ -30,8 +30,8 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/cs3org/reva/internal/http/services/accounts/config" - "github.com/cs3org/reva/internal/http/services/accounts/data" + "github.com/cs3org/reva/internal/http/services/siteacc/config" + "github.com/cs3org/reva/internal/http/services/siteacc/data" "github.com/cs3org/reva/pkg/mentix/key" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -48,7 +48,7 @@ type svc struct { } const ( - serviceName = "accounts" + serviceName = "siteacc" ) // Close is called when this service is being stopped. @@ -341,7 +341,7 @@ func applyDefaultConfig(conf *config.Configuration) { } } -// New returns a new Accounts service. +// New returns a new Site Accounts service. func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) @@ -352,7 +352,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) // Create the accounts manager instance mngr, err := newManager(conf, log) if err != nil { - return nil, errors.Wrap(err, "error creating the accounts service") + return nil, errors.Wrap(err, "error creating the site accounts service") } // Create the service diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go index e542359d9a..bf43a53ceb 100755 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ b/pkg/mentix/exchangers/importers/sitereg/query.go @@ -47,7 +47,7 @@ func decodeQueryData(data []byte) (*siteRegistrationData, error) { return siteData, nil } -func extractQueryInformation(params url.Values) (key.SiteIdentifier, int8, string, error) { +func extractQueryInformation(params url.Values) (key.SiteIdentifier, int, string, error) { apiKey := params.Get("apiKey") if len(apiKey) == 0 { return "", 0, "", errors.Errorf("no API key specified") diff --git a/pkg/mentix/key/apikey.go b/pkg/mentix/key/apikey.go index 3e924240fa..4157e08ff0 100644 --- a/pkg/mentix/key/apikey.go +++ b/pkg/mentix/key/apikey.go @@ -33,9 +33,9 @@ type APIKey = string const ( // FlagDefault marks API keys for default (community) accounts. - FlagDefault int8 = 0x0000 + FlagDefault = 0x0000 // FlagScienceMesh marks API keys for ScienceMesh (partner) accounts. - FlagScienceMesh int8 = 0x0001 + FlagScienceMesh = 0x0001 ) const ( @@ -48,7 +48,7 @@ const ( // GenerateAPIKey generates a new (random) API key which also contains flags and a (salted) hash. // An API key has the following format: // -func GenerateAPIKey(salt string, flags int8) (APIKey, error) { +func GenerateAPIKey(salt string, flags int) (APIKey, error) { if len(salt) == 0 { return "", errors.Errorf("no salt specified") } @@ -79,7 +79,7 @@ func VerifyAPIKey(apiKey APIKey, salt string) error { } // SplitAPIKey splits an API key into its pieces: RandomString, Flags and Hash. -func SplitAPIKey(apiKey APIKey) (string, int8, string, error) { +func SplitAPIKey(apiKey APIKey) (string, int, string, error) { if len(apiKey) != apiKeyLength { return "", 0, "", errors.Errorf("invalid API key length") } @@ -91,10 +91,10 @@ func SplitAPIKey(apiKey APIKey) (string, int8, string, error) { } hash := apiKey[randomStringLength+2:] - return randomString, int8(flags), hash, nil + return randomString, flags, hash, nil } -func calculateHash(randomString string, flags int8, salt string) hashpkg.Hash { +func calculateHash(randomString string, flags int, salt string) hashpkg.Hash { hash := md5.New() _, _ = hash.Write([]byte(randomString)) _, _ = hash.Write([]byte(salt))