[full-ci] enhancement: use reva client pool selectors

.drone.star

@@ -2144,7 +2144,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "IDM_ADMIN_PASSWORD": "admin",  # override the random admin password from `ocis init`
             "FRONTEND_SEARCH_MIN_LENGTH": "2",
             "GATEWAY_GRPC_ADDR": "0.0.0.0:9142",  # make gateway available to wopi server
-            "APP_PROVIDER_EXTERNAL_ADDR": "127.0.0.1:9164",
+            "APP_PROVIDER_EXTERNAL_ADDR": "com.owncloud.api.app-provider",
             "APP_PROVIDER_DRIVER": "wopi",
             "APP_PROVIDER_WOPI_APP_NAME": "FakeOffice",
             "APP_PROVIDER_WOPI_APP_URL": "http://fakeoffice:8080",

changelog/unreleased/client-selector-pool.md

@@ -0,0 +1,8 @@
+Enhancement: Use reva client selectors
+
+Use reva client selectors instead of the static clients, this introduces the ocis service registry in reva.
+The service discovery now resolves reva services by name and the client selectors pick a random registered service node.
+
+https://github.com/owncloud/ocis/pull/6452
+https://github.com/cs3org/reva/pull/3939
+https://github.com/cs3org/reva/pull/3953

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d
-	github.com/cs3org/reva/v2 v2.14.0
+	github.com/cs3org/reva/v2 v2.14.1-0.20230607220921-238a03c2f795
 	github.com/disintegration/imaging v1.6.2
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
 	github.com/egirna/icap-client v0.1.1

go.sum

@@ -629,8 +629,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
 github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
 github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc=
 github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA=
-github.com/cs3org/reva/v2 v2.14.0 h1:X5da4chnEPzqUb76y/DDDawFloRAG7Gy/BMpeYh7vu8=
-github.com/cs3org/reva/v2 v2.14.0/go.mod h1:vMQqSn30fEPHO/GKC2WmGimlOPqvfSy4gdhRSpbvrWc=
+github.com/cs3org/reva/v2 v2.14.1-0.20230607220921-238a03c2f795 h1:uyzA03PcmG7mjd+3KJrkws0IXuXQCvHEn25xXBmO2QI=
+github.com/cs3org/reva/v2 v2.14.1-0.20230607220921-238a03c2f795/go.mod h1:E32krZG159YflDSjDWfx/QGIC2529PS5LiPnGNHu3d0=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

ocis-pkg/registry/register.go

@@ -0,0 +1,47 @@
+package registry
+
+import (
+	"context"
+	"time"
+
+	"github.com/owncloud/ocis/v2/ocis-pkg/log"
+	mRegistry "go-micro.dev/v4/registry"
+)
+
+// RegisterService publishes an arbitrary endpoint to the service-registry. This allows querying nodes of
+// non-micro services like reva. No health-checks are done, thus the caller is responsible for canceling.
+func RegisterService(ctx context.Context, service *mRegistry.Service, logger log.Logger) error {
+	registry := GetRegistry()
+	node := service.Nodes[0]
+
+	logger.Info().Msgf("registering external service %v@%v", node.Id, node.Address)
+
+	rOpts := []mRegistry.RegisterOption{mRegistry.RegisterTTL(time.Minute)}
+	if err := registry.Register(service, rOpts...); err != nil {
+		logger.Fatal().Err(err).Msgf("Registration error for external service %v", service.Name)
+	}
+
+	t := time.NewTicker(time.Second * 30)
+
+	go func() {
+		for {
+			select {
+			case <-t.C:
+				logger.Debug().Interface("service", service).Msg("refreshing external service-registration")
+				err := registry.Register(service, rOpts...)
+				if err != nil {
+					logger.Error().Err(err).Msgf("registration error for external service %v", service.Name)
+				}
+			case <-ctx.Done():
+				logger.Debug().Interface("service", service).Msg("unregistering")
+				t.Stop()
+				err := registry.Deregister(service)
+				if err != nil {
+					logger.Err(err).Msgf("Error unregistering external service %v", service.Name)
+				}
+			}
+		}
+	}()
+
+	return nil
+}

ocis-pkg/service/external/external_test.go → ocis-pkg/registry/register_test.go

@@ -1,4 +1,4 @@
-package external
+package registry
 
 //
 //import (

ocis-pkg/registry/registry.go

@@ -6,14 +6,14 @@ import (
 	"sync"
 	"time"
 
+	rRegistry "github.com/cs3org/reva/v2/pkg/registry"
 	consulr "github.com/go-micro/plugins/v4/registry/consul"
 	etcdr "github.com/go-micro/plugins/v4/registry/etcd"
 	kubernetesr "github.com/go-micro/plugins/v4/registry/kubernetes"
 	mdnsr "github.com/go-micro/plugins/v4/registry/mdns"
 	memr "github.com/go-micro/plugins/v4/registry/memory"
 	natsr "github.com/go-micro/plugins/v4/registry/nats"
-
-	"go-micro.dev/v4/registry"
+	mRegistry "go-micro.dev/v4/registry"
 	"go-micro.dev/v4/registry/cache"
 )
 
@@ -25,7 +25,7 @@ const (
 var (
 	once      sync.Once
 	regPlugin string
-	reg       registry.Registry
+	reg       mRegistry.Registry
 )
 
 func Configure(plugin string) {
@@ -37,7 +37,7 @@ func Configure(plugin string) {
 // GetRegistry returns a configured micro registry based on Micro env vars.
 // It defaults to mDNS, so mind that systems with mDNS disabled by default (i.e SUSE) will have a hard time
 // and it needs to explicitly use etcd. Os awareness for providing a working registry out of the box should be done.
-func GetRegistry() registry.Registry {
+func GetRegistry() mRegistry.Registry {
 	once.Do(func() {
 		addresses := strings.Split(os.Getenv(registryAddressEnv), ",")
 		// prefer env of setting from Configure()
@@ -49,29 +49,33 @@ func GetRegistry() registry.Registry {
 		switch plugin {
 		case "nats":
 			reg = natsr.NewRegistry(
-				registry.Addrs(addresses...),
+				mRegistry.Addrs(addresses...),
 			)
 		case "kubernetes":
 			reg = kubernetesr.NewRegistry(
-				registry.Addrs(addresses...),
+				mRegistry.Addrs(addresses...),
 			)
 		case "etcd":
 			reg = etcdr.NewRegistry(
-				registry.Addrs(addresses...),
+				mRegistry.Addrs(addresses...),
 			)
 		case "consul":
 			reg = consulr.NewRegistry(
-				registry.Addrs(addresses...),
+				mRegistry.Addrs(addresses...),
 			)
 		case "memory":
 			reg = memr.NewRegistry()
 		default:
 			reg = mdnsr.NewRegistry()
 		}
+
 		// No cache needed for in-memory registry
 		if plugin != "memory" {
-			reg = cache.New(reg, cache.WithTTL(20*time.Second))
+			reg = cache.New(reg, cache.WithTTL(30*time.Second))
 		}
+
+		// fixme: lazy initialization of reva registry, needs refactor to a explicit call per service
+		_ = rRegistry.Init(reg)
 	})
 	// always use cached registry to prevent registry
 	// lookup for every request

ocis-pkg/registry/service.go

@@ -0,0 +1,83 @@
+package registry
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	mRegistry "go-micro.dev/v4/registry"
+	"go-micro.dev/v4/util/addr"
+)
+
+func BuildGRPCService(serviceID, uuid, address string, version string) *mRegistry.Service {
+	var host string
+	var port int
+
+	parts := strings.Split(address, ":")
+	if len(parts) > 1 {
+		host = strings.Join(parts[:len(parts)-1], ":")
+		port, _ = strconv.Atoi(parts[len(parts)-1])
+	} else {
+		host = parts[0]
+	}
+
+	addr, err := addr.Extract(host)
+	if err != nil {
+		addr = host
+	}
+
+	node := &mRegistry.Node{
+		Id:       serviceID + "-" + uuid,
+		Address:  net.JoinHostPort(addr, fmt.Sprint(port)),
+		Metadata: make(map[string]string),
+	}
+
+	node.Metadata["registry"] = GetRegistry().String()
+	node.Metadata["server"] = "grpc"
+	node.Metadata["transport"] = "grpc"
+	node.Metadata["protocol"] = "grpc"
+
+	return &mRegistry.Service{
+		Name:      serviceID,
+		Version:   version,
+		Nodes:     []*mRegistry.Node{node},
+		Endpoints: make([]*mRegistry.Endpoint, 0),
+	}
+}
+
+func BuildHTTPService(serviceID, uuid, address string, version string) *mRegistry.Service {
+	var host string
+	var port int
+
+	parts := strings.Split(address, ":")
+	if len(parts) > 1 {
+		host = strings.Join(parts[:len(parts)-1], ":")
+		port, _ = strconv.Atoi(parts[len(parts)-1])
+	} else {
+		host = parts[0]
+	}
+
+	addr, err := addr.Extract(host)
+	if err != nil {
+		addr = host
+	}
+
+	node := &mRegistry.Node{
+		Id:       serviceID + "-" + uuid,
+		Address:  net.JoinHostPort(addr, fmt.Sprint(port)),
+		Metadata: make(map[string]string),
+	}
+
+	node.Metadata["registry"] = GetRegistry().String()
+	node.Metadata["server"] = "http"
+	node.Metadata["transport"] = "http"
+	node.Metadata["protocol"] = "http"
+
+	return &mRegistry.Service{
+		Name:      serviceID,
+		Version:   version,
+		Nodes:     []*mRegistry.Node{node},
+		Endpoints: make([]*mRegistry.Endpoint, 0),
+	}
+}

ocis-pkg/shared/reva.go

@@ -1,9 +1,11 @@
 package shared
 
-import "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
+import (
+	"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
+)
 
 var defaultRevaConfig = Reva{
-	Address: "127.0.0.1:9142",
+	Address: "com.owncloud.api.gateway",
 }
 
 func DefaultRevaConfig() *Reva {