From e37b10e7e2a20167f865c51f7fcb33385fcede29 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Fri, 16 Apr 2021 13:07:44 +0200 Subject: [PATCH 1/8] named registries --- changelog/unreleased/named-services.md | 5 + cmd/revad/runtime/option.go | 13 +- cmd/revad/runtime/runtime.go | 20 ++ examples/ocmd/ocmd-server-1.toml | 11 ++ go.mod | 1 + go.sum | 1 + internal/grpc/services/gateway/gateway.go | 5 +- pkg/registry/config.go | 53 ++++++ pkg/registry/memory/memory.go | 116 ++++++++++++ pkg/registry/memory/memory_test.go | 176 ++++++++++++++++++ pkg/registry/memory/node.go | 44 +++++ pkg/registry/memory/service.go | 56 ++++++ pkg/registry/registry.go | 49 +++++ pkg/rgrpc/todo/pool/pool.go | 20 +- pkg/utils/utils.go | 5 + tests/oc-integration-tests/drone/gateway.toml | 7 + .../drone/storage-home-ocis.toml | 7 + 17 files changed, 585 insertions(+), 4 deletions(-) create mode 100644 changelog/unreleased/named-services.md create mode 100644 pkg/registry/config.go create mode 100644 pkg/registry/memory/memory.go create mode 100644 pkg/registry/memory/memory_test.go create mode 100644 pkg/registry/memory/node.go create mode 100644 pkg/registry/memory/service.go create mode 100644 pkg/registry/registry.go diff --git a/changelog/unreleased/named-services.md b/changelog/unreleased/named-services.md new file mode 100644 index 0000000000..057f22cdb0 --- /dev/null +++ b/changelog/unreleased/named-services.md @@ -0,0 +1,5 @@ +Enhancement: Named Service Registration + +move away from hardcoding service IP addresses and rely upon name resolution instead. It delegates the address lookup to a static in-memory service registry, which can be re-implemented in multiple forms. + +https://github.com/cs3org/reva/pull/1509 \ No newline at end of file diff --git a/cmd/revad/runtime/option.go b/cmd/revad/runtime/option.go index 3e1dee98f8..1b4084cb7b 100644 --- a/cmd/revad/runtime/option.go +++ b/cmd/revad/runtime/option.go @@ -19,6 +19,7 @@ package runtime import ( + "github.com/cs3org/reva/pkg/registry" "github.com/rs/zerolog" ) @@ -27,10 +28,11 @@ type Option func(o *Options) // Options defines the available options for this package. type Options struct { - Logger *zerolog.Logger + Logger *zerolog.Logger + Registry registry.Registry } -// newOptions intializes the available default options. +// newOptions initializes the available default options. func newOptions(opts ...Option) Options { opt := Options{} @@ -47,3 +49,10 @@ func WithLogger(logger *zerolog.Logger) Option { o.Logger = logger } } + +// WithRegistry provides a function to set the registry. +func WithRegistry(r registry.Registry) Option { + return func(o *Options) { + o.Registry = r + } +} diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 7c7e6a6e23..b1e918b102 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -28,6 +28,10 @@ import ( "strconv" "strings" + "github.com/cs3org/reva/pkg/registry/memory" + + "github.com/cs3org/reva/pkg/utils" + "contrib.go.opencensus.io/exporter/jaeger" "github.com/cs3org/reva/cmd/revad/internal/grace" "github.com/cs3org/reva/pkg/logger" @@ -56,6 +60,21 @@ func RunWithOptions(mainConf map[string]interface{}, pidFile string, opts ...Opt parseSharedConfOrDie(mainConf["shared"]) coreConf := parseCoreConfOrDie(mainConf["core"]) + // TODO: one can pass the options from the config file to registry.New() and initialize a registry based upon config files. + if options.Registry != nil { + utils.GlobalRegistry = options.Registry + } else if _, ok := mainConf["registry"]; ok { + for _, services := range mainConf["registry"].(map[string]interface{}) { + for sName, nodes := range services.(map[string]interface{}) { + for _, instance := range nodes.([]interface{}) { + if err := utils.GlobalRegistry.Add(memory.NewService(sName, instance.(map[string]interface{})["nodes"].([]interface{}))); err != nil { + panic(err) + } + } + } + } + } + run(mainConf, coreConf, options.Logger, pidFile) } @@ -71,6 +90,7 @@ func run(mainConf map[string]interface{}, coreConf *coreConf, logger *zerolog.Lo host, _ := os.Hostname() logger.Info().Msgf("host info: %s", host) + // initRegistry() initTracing(coreConf, logger) initCPUCount(coreConf, logger) diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index 9d0335ced0..87c7fb394e 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -1,6 +1,17 @@ [shared] gatewaysvc = "localhost:19000" +[registry] +driver = "static" + +[registry.static] +services = ["authprovider","userprovider"] + +[registry.static.authprovider] +bearer = ["localhost:0123"] +basic = ["localhost:1234"] +publiclink = ["localhost:9876"] + [grpc] address = "0.0.0.0:19000" diff --git a/go.mod b/go.mod index 5c33a7b29b..47b14873ec 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 google.golang.org/grpc v1.37.0 google.golang.org/protobuf v1.26.0 + gotest.tools v2.2.0+incompatible ) go 1.16 diff --git a/go.sum b/go.sum index 767f754d0b..96d744c4a8 100644 --- a/go.sum +++ b/go.sum @@ -1529,6 +1529,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 0388c441d1..6260e0ee45 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -82,7 +82,10 @@ func (c *config) init() { // if services address are not specified we used the shared conf // for the gatewaysvc to have dev setups very quickly. - c.AuthRegistryEndpoint = sharedconf.GetGatewaySVC(c.AuthRegistryEndpoint) + + // we're commenting this line to showcase the fact that now we don't want to point to an ip address but rather + // resolve an ip address from a name. + // c.AuthRegistryEndpoint = sharedconf.GetGatewaySVC(c.AuthRegistryEndpoint) c.StorageRegistryEndpoint = sharedconf.GetGatewaySVC(c.StorageRegistryEndpoint) c.AppRegistryEndpoint = sharedconf.GetGatewaySVC(c.AppRegistryEndpoint) c.PreferencesEndpoint = sharedconf.GetGatewaySVC(c.PreferencesEndpoint) diff --git a/pkg/registry/config.go b/pkg/registry/config.go new file mode 100644 index 0000000000..4e0ff51d7d --- /dev/null +++ b/pkg/registry/config.go @@ -0,0 +1,53 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "github.com/mitchellh/mapstructure" +) + +// Config configures a registry +type Config struct { + Services map[string][]*service `mapstructure:"services"` +} + +// service implements the Service interface +type service struct { + Name string `mapstructure:"name"` + Nodes []node `mapstructure:"nodes"` +} + +type node struct { + Address string `mapstructure:"address"` + Metadata map[string]string `mapstructure:"metadata"` +} + +// ParseConfig translates Config file values into a Config struct for consumers. +func ParseConfig(m map[string]interface{}) (*Config, error) { + c := &Config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + + if len(c.Services) == 0 { + c.Services = make(map[string][]*service) + } + + return c, nil +} diff --git a/pkg/registry/memory/memory.go b/pkg/registry/memory/memory.go new file mode 100644 index 0000000000..929083ef95 --- /dev/null +++ b/pkg/registry/memory/memory.go @@ -0,0 +1,116 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package memory + +import ( + "fmt" + "sync" + + "github.com/cs3org/reva/pkg/registry" +) + +// Registry implements the Registry interface. +type Registry struct { + // m protects async access to the services map. + sync.Mutex + // services map a service name with a set of nodes. + services map[string][]registry.Service +} + +// Add implements the Registry interface. If the service is already known in this registry it will only update the nodes. +func (r *Registry) Add(svc registry.Service) error { + r.Lock() + defer r.Unlock() + + // init the new service in the registry storage if we have not seen it before + if _, ok := r.services[svc.Name()]; !ok { + s := []registry.Service{svc} + r.services[svc.Name()] = s + return nil + } + + // append the requested nodes to the existing service + for i := range r.services[svc.Name()] { + ns := make([]node, 0) + nodes := append(r.services[svc.Name()][i].Nodes(), svc.Nodes()...) + for n := range nodes { + ns = append(ns, node{ + id: nodes[n].ID(), + address: nodes[n].Address(), + metadata: nodes[n].Metadata(), + }) + } + r.services[svc.Name()] = []registry.Service{ + service{ + svc.Name(), + ns, + }, + } + fmt.Printf("%+v\n", r.services[svc.Name()]) + } + + return nil +} + +// GetService implements the Registry interface. There is currently no load balance being done, but it should not be +// hard to add. +func (r *Registry) GetService(name string) ([]registry.Service, error) { + r.Lock() + defer r.Unlock() + + if services, ok := r.services[name]; ok { + return services, nil + } + + return nil, fmt.Errorf("service %v not found", name) +} + +// New returns an implementation of the Registry interface. +func New(m map[string]interface{}) registry.Registry { + c, err := registry.ParseConfig(m) + if err != nil { + return nil + } + + r := make(map[string][]registry.Service) + for sKey, services := range c.Services { + // allocate as much memory as total services were parsed from config + r[sKey] = make([]registry.Service, len(services)) + + for i := range services { + s := service{ + name: services[i].Name, + } + + // copy nodes + for j := range services[i].Nodes { + s.nodes = append(s.nodes, node{ + address: services[i].Nodes[j].Address, + metadata: services[i].Nodes[j].Metadata, + }) + } + + r[sKey][i] = &s + } + } + + return &Registry{ + services: r, + } +} diff --git a/pkg/registry/memory/memory_test.go b/pkg/registry/memory/memory_test.go new file mode 100644 index 0000000000..fbdb708637 --- /dev/null +++ b/pkg/registry/memory/memory_test.go @@ -0,0 +1,176 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package memory + +import ( + "fmt" + "os" + "testing" + + "github.com/cs3org/reva/pkg/registry" + + "github.com/google/uuid" + "gotest.tools/assert" +) + +var ( + in = make(map[string]interface{}) + reg = New(in) + node1 = node{ + id: uuid.New().String(), + address: "0.0.0.0:42069", + metadata: map[string]string{ + "type": "auth-bearer", + }, + } + + node2 = node{ + id: uuid.New().String(), + address: "0.0.0.0:7777", + metadata: map[string]string{ + "type": "auth-basic", + }, + } + + node3 = node{id: uuid.NewString(), address: "0.0.0.0:8888"} + node4 = node{id: uuid.NewString(), address: "0.0.0.0:9999"} +) + +var scenarios = []struct { + name string // scenario name + in string // used to query the Registry by service name + services []service + expectedNodes []node // expected set of nodes +}{ + { + name: "single service with 2 nodes", + in: "auth-provider", + services: []service{ + {name: "auth-provider", nodes: []node{node1, node2}}, + }, + expectedNodes: []node{node1, node2}, + }, + { + name: "single service with 2 nodes scaled x2", + in: "auth-provider", + services: []service{ + {name: "auth-provider", nodes: []node{node1, node2}}, + {name: "auth-provider", nodes: []node{node3, node4}}, + }, + expectedNodes: []node{node1, node2, node3, node4}, + }, +} + +func TestAdd(t *testing.T) { + reg = New(in) + s1 := scenarios[1].services[0] + s2 := scenarios[1].services[1] + _ = reg.Add(s1) + _ = reg.Add(s2) + + _ = reg.Add(service{ + name: "test", + nodes: []node{ + { + id: "1234", + address: "localhost:8899", + metadata: nil, + }, + }, + }) + + expectedNumberOfNodes := len(s1.nodes) + len(s2.nodes) + if s, err := reg.GetService(s1.name); err != nil { + t.Error(err) + collectedNumberOfNodes := 0 + if len(s) != 0 { + for _, service := range s { + collectedNumberOfNodes += len(service.Nodes()) + } + } else { + t.Error(fmt.Errorf("invalid number of registered services: 0")) + } + + if expectedNumberOfNodes == collectedNumberOfNodes { + t.Error(fmt.Errorf("expected %v nodes, got: %v", expectedNumberOfNodes, collectedNumberOfNodes)) + } + } +} + +func TestGetService(t *testing.T) { + for _, scenario := range scenarios { + reg = New(in) + for _, service := range scenario.services { + if err := reg.Add(&service); err != nil { + os.Exit(1) + } + } + t.Run(scenario.name, func(t *testing.T) { + svc, err := reg.GetService(scenario.in) + if err != nil { + t.Error(err) + } + + totalNodes := 0 + for i := range svc { + totalNodes += len(svc[i].Nodes()) + for _, node := range svc[i].Nodes() { + if !contains(svc[i].Nodes(), node) { + t.Errorf("unexpected return value: Registry does not contain node %s", node) + } + } + } + assert.Equal(t, len(scenario.expectedNodes), totalNodes) + }) + } +} + +func TestGetServiceInDepth(t *testing.T) { + for _, scenario := range scenarios { + // restart Registry + reg = New(in) + // register all services + for i := range scenario.services { + if err := reg.Add(&scenario.services[i]); err != nil { + os.Exit(1) + } + } + t.Run(scenario.name, func(t *testing.T) { + services, err := reg.GetService(scenario.in) + if err != nil { + t.Error(err) + } + + for _, service := range services { + for _, nodes := range service.Nodes() { + fmt.Println(nodes.Address()) + } + } + }) + } +} + +func contains(a []registry.Node, b registry.Node) bool { + for i := range a { + if a[i].Address() == b.Address() { + return true + } + } + return false +} diff --git a/pkg/registry/memory/node.go b/pkg/registry/memory/node.go new file mode 100644 index 0000000000..22042306dd --- /dev/null +++ b/pkg/registry/memory/node.go @@ -0,0 +1,44 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package memory + +import "fmt" + +// node implements the registry.Node interface. +type node struct { + id string + address string + metadata map[string]string +} + +func (n node) Address() string { + return n.address +} + +func (n node) Metadata() map[string]string { + return n.metadata +} + +func (n node) String() string { + return fmt.Sprintf("%v-%v", n.id, n.address) +} + +func (n node) ID() string { + return n.id +} diff --git a/pkg/registry/memory/service.go b/pkg/registry/memory/service.go new file mode 100644 index 0000000000..683e33dadc --- /dev/null +++ b/pkg/registry/memory/service.go @@ -0,0 +1,56 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package memory + +import "github.com/cs3org/reva/pkg/registry" + +func NewService(name string, nodes []interface{}) registry.Service { + n := make([]node, 0) + for i := 0; i < len(nodes); i++ { + n = append(n, node{ + // explicit type conversions because types are not exported to prevent from circular dependencies until released. + id: nodes[i].(map[string]interface{})["id"].(string), + address: nodes[i].(map[string]interface{})["address"].(string), + //metadata: nodes[i].(map[string]interface{})["metadata"].(map[string]string), + }) + } + + return service{ + name: name, + nodes: n, + } +} + +// service implements the Service interface +type service struct { + name string + nodes []node +} + +func (s service) Name() string { + return s.name +} + +func (s service) Nodes() []registry.Node { + ret := make([]registry.Node, 0) + for i := range s.nodes { + ret = append(ret, s.nodes[i]) + } + return ret +} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 0000000000..4084a93ebb --- /dev/null +++ b/pkg/registry/registry.go @@ -0,0 +1,49 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +// Registry provides with means for dynamically registering services. +type Registry interface { + // Add registers a Service on the memoryRegistry. Repeated names is allowed, services are distinguished by their metadata. + Add(Service) error + + // GetService retrieves a Service and all of its nodes by Service name. It returns []*Service because we can have + // multiple versions of the same Service running alongside each others. + GetService(string) ([]Service, error) +} + +// Service defines a service. +type Service interface { + Name() string + Nodes() []Node +} + +// Node defines nodes on a service. +type Node interface { + // Address where the given node is running. + Address() string + + // metadata is used in order to differentiate services implementations. For instance an AuthProvider Service could + // have multiple implementations, basic, bearer ..., metadata would be used to select the Service type depending on + // its implementation. + Metadata() map[string]string + + // ID returns the node ID. + ID() string +} diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 956847bf16..4c0efb4471 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -38,7 +38,6 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storageregistry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" - "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" ) @@ -171,15 +170,18 @@ func GetAuthRegistryServiceClient(endpoint string) (authregistry.RegistryAPIClie authRegistries.m.Lock() defer authRegistries.m.Unlock() + // if there is already a connection to this node, use it. if c, ok := authRegistries.conn[endpoint]; ok { return c.(authregistry.RegistryAPIClient), nil } + // if not, create a new connection conn, err := NewConn(endpoint) if err != nil { return nil, err } + // and memoize it v := authregistry.NewRegistryAPIClient(conn) authRegistries.conn[endpoint] = v return v, nil @@ -412,3 +414,19 @@ func GetDataTxClient(endpoint string) (datatx.TxAPIClient, error) { dataTxs.conn[endpoint] = v return v, nil } + +// getEndpointByName resolve service names to ip addresses present on the registry. +// func getEndpointByName(name string) (string, error) { +// if services, err := utils.GlobalRegistry.GetService(name); err == nil { +// if len(services) > 0 { +// for i := range services { +// for j := range services[i].Nodes() { +// // return the first one. This MUST be improved upon with selectors. +// return services[i].Nodes()[j].Address(), nil +// } +// } +// } +// } +// +// return "", fmt.Errorf("could not get service by name: %v", name) +// } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 88b50aa785..deeaa0720a 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -32,6 +32,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/registry" + "github.com/cs3org/reva/pkg/registry/memory" "github.com/golang/protobuf/proto" "google.golang.org/protobuf/encoding/protojson" ) @@ -40,6 +42,9 @@ var ( matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") matchEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") + // GlobalRegistry configures a service registry globally accessible. It defaults to a memory registry. The usage of + // globals is not encouraged, and this is a workaround until the PR is out of a draft state. + GlobalRegistry registry.Registry = memory.New(map[string]interface{}{}) ) // Skip evaluates whether a source endpoint contains any of the prefixes. diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index a535422b3e..dfa249238d 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -2,6 +2,13 @@ jwt_secret = "Pive-Fumkiu4" gatewaysvc = "localhost:19000" +#[[registry.services.storagehome]] +#name = 'authregistry' +# +#[[registry.services.storagehome.nodes]] +#id = '65ad3270-9e87-11eb-a1aa-0fcc1edaa55e' +#address = '0.0.0.0:9142' + # This gateway.toml config file will start a reva service that: # - serves as a gateway for all requests # - looks up the storageprovider using a storageregistry diff --git a/tests/oc-integration-tests/drone/storage-home-ocis.toml b/tests/oc-integration-tests/drone/storage-home-ocis.toml index cabc334327..d51c509af7 100644 --- a/tests/oc-integration-tests/drone/storage-home-ocis.toml +++ b/tests/oc-integration-tests/drone/storage-home-ocis.toml @@ -3,6 +3,13 @@ jwt_secret = "Pive-Fumkiu4" gatewaysvc = "localhost:19000" +#[[registry.services.storagehome]] +#name = 'storage-home' +# +#[[registry.services.storagehome.nodes]] +#id = '130e0018-9e86-11eb-8634-336624ad2203' +#address = '0.0.0.0:9154' + # - authenticates grpc storage provider requests using the internal jwt token # - authenticates http upload and download requests requests using basic auth # - serves the home storage provider on grpc port 12000 From 58d013a7509d1941834e1bc814e9a9fa8bff00b1 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Mon, 19 Apr 2021 11:10:23 +0200 Subject: [PATCH 2/8] comment exported methods --- pkg/registry/memory/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/registry/memory/service.go b/pkg/registry/memory/service.go index 683e33dadc..d2d5aed20e 100644 --- a/pkg/registry/memory/service.go +++ b/pkg/registry/memory/service.go @@ -20,6 +20,7 @@ package memory import "github.com/cs3org/reva/pkg/registry" +// NewService creates a new memory registry.Service. func NewService(name string, nodes []interface{}) registry.Service { n := make([]node, 0) for i := 0; i < len(nodes); i++ { @@ -43,10 +44,12 @@ type service struct { nodes []node } +// Name implements the service interface. func (s service) Name() string { return s.name } +// Nodes implements the service interface. func (s service) Nodes() []registry.Node { ret := make([]registry.Node, 0) for i := range s.nodes { From 5e2c1549b6fb9c70589149b4e157c38be872c532 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 11:10:50 +0200 Subject: [PATCH 3/8] requested changes: use map[string]map[string]*service for config; use map[string]registry.Service instead of map[string][]registry.Service for the memory implementation --- pkg/registry/config.go | 6 +-- pkg/registry/memory/memory.go | 73 ++++++++---------------------- pkg/registry/memory/memory_test.go | 64 +++++--------------------- pkg/registry/memory/service.go | 11 +++++ pkg/registry/registry.go | 2 +- 5 files changed, 46 insertions(+), 110 deletions(-) diff --git a/pkg/registry/config.go b/pkg/registry/config.go index 4e0ff51d7d..5f3a930e7b 100644 --- a/pkg/registry/config.go +++ b/pkg/registry/config.go @@ -24,10 +24,10 @@ import ( // Config configures a registry type Config struct { - Services map[string][]*service `mapstructure:"services"` + Services map[string]map[string]*Service `mapstructure:"services"` } -// service implements the Service interface +// service implements the Service interface. Attributes are exported so that mapstructure can unmarshal values onto them. type service struct { Name string `mapstructure:"name"` Nodes []node `mapstructure:"nodes"` @@ -46,7 +46,7 @@ func ParseConfig(m map[string]interface{}) (*Config, error) { } if len(c.Services) == 0 { - c.Services = make(map[string][]*service) + c.Services = make(map[string]map[string]*Service) } return c, nil diff --git a/pkg/registry/memory/memory.go b/pkg/registry/memory/memory.go index 929083ef95..fe9a14fdec 100644 --- a/pkg/registry/memory/memory.go +++ b/pkg/registry/memory/memory.go @@ -30,7 +30,7 @@ type Registry struct { // m protects async access to the services map. sync.Mutex // services map a service name with a set of nodes. - services map[string][]registry.Service + services map[string]registry.Service } // Add implements the Registry interface. If the service is already known in this registry it will only update the nodes. @@ -38,44 +38,31 @@ func (r *Registry) Add(svc registry.Service) error { r.Lock() defer r.Unlock() - // init the new service in the registry storage if we have not seen it before - if _, ok := r.services[svc.Name()]; !ok { - s := []registry.Service{svc} + // append the nodes if the service is already registered. + if _, ok := r.services[svc.Name()]; ok { + s := service{ + name: svc.Name(), + nodes: make([]node, 0), + } + + s.mergeNodes(svc.Nodes(), r.services[svc.Name()].Nodes()) + r.services[svc.Name()] = s return nil } - // append the requested nodes to the existing service - for i := range r.services[svc.Name()] { - ns := make([]node, 0) - nodes := append(r.services[svc.Name()][i].Nodes(), svc.Nodes()...) - for n := range nodes { - ns = append(ns, node{ - id: nodes[n].ID(), - address: nodes[n].Address(), - metadata: nodes[n].Metadata(), - }) - } - r.services[svc.Name()] = []registry.Service{ - service{ - svc.Name(), - ns, - }, - } - fmt.Printf("%+v\n", r.services[svc.Name()]) - } - + r.services[svc.Name()] = svc return nil } // GetService implements the Registry interface. There is currently no load balance being done, but it should not be // hard to add. -func (r *Registry) GetService(name string) ([]registry.Service, error) { +func (r *Registry) GetService(name string) (registry.Service, error) { r.Lock() defer r.Unlock() - if services, ok := r.services[name]; ok { - return services, nil + if service, ok := r.services[name]; ok { + return service, nil } return nil, fmt.Errorf("service %v not found", name) @@ -83,34 +70,12 @@ func (r *Registry) GetService(name string) ([]registry.Service, error) { // New returns an implementation of the Registry interface. func New(m map[string]interface{}) registry.Registry { - c, err := registry.ParseConfig(m) - if err != nil { - return nil - } - - r := make(map[string][]registry.Service) - for sKey, services := range c.Services { - // allocate as much memory as total services were parsed from config - r[sKey] = make([]registry.Service, len(services)) - - for i := range services { - s := service{ - name: services[i].Name, - } - - // copy nodes - for j := range services[i].Nodes { - s.nodes = append(s.nodes, node{ - address: services[i].Nodes[j].Address, - metadata: services[i].Nodes[j].Metadata, - }) - } - - r[sKey][i] = &s - } - } + //c, err := registry.ParseConfig(m) + //if err != nil { + // return nil + //} return &Registry{ - services: r, + services: map[string]registry.Service{}, } } diff --git a/pkg/registry/memory/memory_test.go b/pkg/registry/memory/memory_test.go index fbdb708637..7dd7e00aa7 100644 --- a/pkg/registry/memory/memory_test.go +++ b/pkg/registry/memory/memory_test.go @@ -23,8 +23,6 @@ import ( "os" "testing" - "github.com/cs3org/reva/pkg/registry" - "github.com/google/uuid" "gotest.tools/assert" ) @@ -98,14 +96,7 @@ func TestAdd(t *testing.T) { expectedNumberOfNodes := len(s1.nodes) + len(s2.nodes) if s, err := reg.GetService(s1.name); err != nil { t.Error(err) - collectedNumberOfNodes := 0 - if len(s) != 0 { - for _, service := range s { - collectedNumberOfNodes += len(service.Nodes()) - } - } else { - t.Error(fmt.Errorf("invalid number of registered services: 0")) - } + collectedNumberOfNodes := len(s.Nodes()) if expectedNumberOfNodes == collectedNumberOfNodes { t.Error(fmt.Errorf("expected %v nodes, got: %v", expectedNumberOfNodes, collectedNumberOfNodes)) @@ -121,56 +112,25 @@ func TestGetService(t *testing.T) { os.Exit(1) } } + t.Run(scenario.name, func(t *testing.T) { svc, err := reg.GetService(scenario.in) if err != nil { t.Error(err) } - totalNodes := 0 - for i := range svc { - totalNodes += len(svc[i].Nodes()) - for _, node := range svc[i].Nodes() { - if !contains(svc[i].Nodes(), node) { - t.Errorf("unexpected return value: Registry does not contain node %s", node) - } - } - } + totalNodes := len(svc.Nodes()) assert.Equal(t, len(scenario.expectedNodes), totalNodes) }) } } -func TestGetServiceInDepth(t *testing.T) { - for _, scenario := range scenarios { - // restart Registry - reg = New(in) - // register all services - for i := range scenario.services { - if err := reg.Add(&scenario.services[i]); err != nil { - os.Exit(1) - } - } - t.Run(scenario.name, func(t *testing.T) { - services, err := reg.GetService(scenario.in) - if err != nil { - t.Error(err) - } - - for _, service := range services { - for _, nodes := range service.Nodes() { - fmt.Println(nodes.Address()) - } - } - }) - } -} - -func contains(a []registry.Node, b registry.Node) bool { - for i := range a { - if a[i].Address() == b.Address() { - return true - } - } - return false -} +// +//func contains(a []registry.Node, b registry.Node) bool { +// for i := range a { +// if a[i].Address() == b.Address() { +// return true +// } +// } +// return false +//} diff --git a/pkg/registry/memory/service.go b/pkg/registry/memory/service.go index d2d5aed20e..0213b2761d 100644 --- a/pkg/registry/memory/service.go +++ b/pkg/registry/memory/service.go @@ -57,3 +57,14 @@ func (s service) Nodes() []registry.Node { } return ret } + +func (s *service) mergeNodes(n1, n2 []registry.Node) { + n1 = append(n1, n2...) + for _, n := range n1 { + s.nodes = append(s.nodes, node{ + id: n.ID(), + address: n.Address(), + metadata: n.Metadata(), + }) + } +} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 4084a93ebb..8b705a0937 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -25,7 +25,7 @@ type Registry interface { // GetService retrieves a Service and all of its nodes by Service name. It returns []*Service because we can have // multiple versions of the same Service running alongside each others. - GetService(string) ([]Service, error) + GetService(string) (Service, error) } // Service defines a service. From cec3e6929e9b8c1d7f9ab2c7c24e83dafb6f98a4 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 11:38:07 +0200 Subject: [PATCH 4/8] fix linter --- pkg/registry/config.go | 4 ++-- pkg/registry/memory/memory.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/registry/config.go b/pkg/registry/config.go index 5f3a930e7b..f798b65b9c 100644 --- a/pkg/registry/config.go +++ b/pkg/registry/config.go @@ -24,7 +24,7 @@ import ( // Config configures a registry type Config struct { - Services map[string]map[string]*Service `mapstructure:"services"` + Services map[string]map[string]*service `mapstructure:"services"` } // service implements the Service interface. Attributes are exported so that mapstructure can unmarshal values onto them. @@ -46,7 +46,7 @@ func ParseConfig(m map[string]interface{}) (*Config, error) { } if len(c.Services) == 0 { - c.Services = make(map[string]map[string]*Service) + c.Services = make(map[string]map[string]*service) } return c, nil diff --git a/pkg/registry/memory/memory.go b/pkg/registry/memory/memory.go index fe9a14fdec..6d828ec6c1 100644 --- a/pkg/registry/memory/memory.go +++ b/pkg/registry/memory/memory.go @@ -70,10 +70,10 @@ func (r *Registry) GetService(name string) (registry.Service, error) { // New returns an implementation of the Registry interface. func New(m map[string]interface{}) registry.Registry { - //c, err := registry.ParseConfig(m) - //if err != nil { + // c, err := registry.ParseConfig(m) + // if err != nil { // return nil - //} + // } return &Registry{ services: map[string]registry.Service{}, From cfc5eb200b0b1e89fc58a1b08da861e324a091bd Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 12:43:59 +0200 Subject: [PATCH 5/8] add config tests --- pkg/registry/config_test.go | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 pkg/registry/config_test.go diff --git a/pkg/registry/config_test.go b/pkg/registry/config_test.go new file mode 100644 index 0000000000..f2d632f536 --- /dev/null +++ b/pkg/registry/config_test.go @@ -0,0 +1,74 @@ +package registry + +import ( + "reflect" + "testing" +) + +func TestParseConfig(t *testing.T) { + type args struct { + m map[string]interface{} + } + tests := []struct { + name string + args args + want *Config + wantErr bool + }{ + {name: "parse config", args: args{map[string]interface{}{ + "services": map[string]map[string]interface{}{ + "authprovider": map[string]interface{}{ + "basic": map[string]interface{}{ + "name": "auth-basic", + "nodes": []map[string]interface{}{ + { + "address": "0.0.0.0:1234", + "metadata": map[string]string{"version": "v0.1.0"}, + }, + }, + }, + "bearer": map[string]interface{}{ + "name": "auth-bearer", + "nodes": []map[string]interface{}{ + { + "address": "0.0.0.0:5678", + "metadata": map[string]string{"version": "v0.1.0"}, + }, + }, + }, + }, + }, + }}, want: &Config{ + Services: map[string]map[string]*service{ + "authprovider": map[string]*service{ + "basic": &service{ + Name: "auth-basic", + Nodes: []node{{ + Address: "0.0.0.0:1234", + Metadata: map[string]string{"version": "v0.1.0"}, + }}, + }, + "bearer": &service{ + Name: "auth-bearer", + Nodes: []node{{ + Address: "0.0.0.0:5678", + Metadata: map[string]string{"version": "v0.1.0"}, + }}, + }, + }, + }, + }, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseConfig(tt.args.m) + if (err != nil) != tt.wantErr { + t.Errorf("ParseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseConfig() got = %v, want %v", got, tt.want) + } + }) + } +} From 6ef9dd1e1d267f2a162c3a14546b4bb94235d956 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 12:45:22 +0200 Subject: [PATCH 6/8] fix linter and add license header --- pkg/registry/config_test.go | 18 ++++++++++++++++++ pkg/registry/memory/memory_test.go | 13 ++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/registry/config_test.go b/pkg/registry/config_test.go index f2d632f536..90c648670b 100644 --- a/pkg/registry/config_test.go +++ b/pkg/registry/config_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package registry import ( diff --git a/pkg/registry/memory/memory_test.go b/pkg/registry/memory/memory_test.go index 7dd7e00aa7..d7c3422564 100644 --- a/pkg/registry/memory/memory_test.go +++ b/pkg/registry/memory/memory_test.go @@ -125,12 +125,11 @@ func TestGetService(t *testing.T) { } } -// -//func contains(a []registry.Node, b registry.Node) bool { -// for i := range a { -// if a[i].Address() == b.Address() { -// return true +// func contains(a []registry.Node, b registry.Node) bool { +// for i := range a { +// if a[i].Address() == b.Address() { +// return true +// } // } +// return false // } -// return false -//} From d54088372205b8cb8bc1c5eac2af4d45a0fa065a Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 12:51:13 +0200 Subject: [PATCH 7/8] add config example --- pkg/registry/config_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/registry/config_test.go b/pkg/registry/config_test.go index 90c648670b..82bc49a49b 100644 --- a/pkg/registry/config_test.go +++ b/pkg/registry/config_test.go @@ -23,6 +23,26 @@ import ( "testing" ) +/* +config example: + +--- +services: + authprovider: + basic: + name: auth-basic + nodes: + - address: 0.0.0.0:1234 + metadata: + version: v0.1.0 + bearer: + name: auth-bearer + nodes: + - address: 0.0.0.0:5678 + metadata: + version: v0.1.0 + +*/ func TestParseConfig(t *testing.T) { type args struct { m map[string]interface{} From f29e81adbec4666b23ff485455bc5ad9ac180590 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Wed, 21 Apr 2021 14:38:25 +0200 Subject: [PATCH 8/8] leftover comment --- internal/grpc/services/gateway/gateway.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 6260e0ee45..560de2a6e2 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -85,7 +85,7 @@ func (c *config) init() { // we're commenting this line to showcase the fact that now we don't want to point to an ip address but rather // resolve an ip address from a name. - // c.AuthRegistryEndpoint = sharedconf.GetGatewaySVC(c.AuthRegistryEndpoint) + c.AuthRegistryEndpoint = sharedconf.GetGatewaySVC(c.AuthRegistryEndpoint) c.StorageRegistryEndpoint = sharedconf.GetGatewaySVC(c.StorageRegistryEndpoint) c.AppRegistryEndpoint = sharedconf.GetGatewaySVC(c.AppRegistryEndpoint) c.PreferencesEndpoint = sharedconf.GetGatewaySVC(c.PreferencesEndpoint)