Enhance storage registry with virtual views and regular expressions. (c…

…s3org#1570)
ffurano · Apr 19, 2021 · 759858a · 759858a
1 parent b0204d3
commit 759858a
Show file tree

Hide file tree

Showing 26 changed files with 696 additions and 202 deletions.
diff --git a/.golangci.yaml b/.golangci.yaml
@@ -35,4 +35,3 @@ linters:
     - gocritic
     - prealloc
     #- gosec
-
diff --git a/changelog/unreleased/storage-registry-refactor.md b/changelog/unreleased/storage-registry-refactor.md
@@ -0,0 +1,9 @@
+Enhancement: Enhance storage registry with virtual views and regular expressions
+
+Add the functionality to the storage registry service to handle user requests
+for references which can span across multiple storage providers, particularly
+useful for cases where directories are sharded across providers or virtual views
+are expected.
+
+https://github.com/cs3org/cs3apis/pull/116
+https://github.com/cs3org/reva/pull/1570
diff --git a/cmd/revad/main.go b/cmd/revad/main.go
@@ -227,8 +227,8 @@ func runMultiple(confs []map[string]interface{}) {
 		wg.Add(1)
 		pidfile := getPidfile()
 		go func(wg *sync.WaitGroup, conf map[string]interface{}) {
+			defer wg.Done()
 			runtime.Run(conf, pidfile, *logFlag)
-			wg.Done()
 		}(&wg, conf)
 	}
 	wg.Wait()

diff --git a/examples/oc-phoenix/gateway.toml b/examples/oc-phoenix/gateway.toml
@@ -48,15 +48,17 @@ driver = "static"
 [grpc.services.storageregistry.drivers.static]
 home_provider = "/home"
 
-[grpc.services.storageregistry.drivers.static.rules]
 # mount a home storage provider that uses a context based path wrapper
 # to jail users into their home dir
-"/home" = "localhost:12000"
+[grpc.services.storageregistry.drivers.static.rules."/home"]
+address = "localhost:12000"
 
 # mount a storage provider without a path wrapper for direct access to users.
-"/oc" = "localhost:11000"
-"123e4567-e89b-12d3-a456-426655440000" = "localhost:11000"
-# another mount point might be "/projects/" 
+[grpc.services.storageregistry.drivers.static.rules."/oc"]
+address = "localhost:11000"
+[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"]
+address = "localhost:11000"
+# another mount point might be "/projects/"
 
 [http]
 address = "0.0.0.0:19001"

diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml
@@ -33,9 +33,10 @@ driver = "static"
 [grpc.services.storageregistry.drivers.static]
 home_provider = "/home"
 
-[grpc.services.storageregistry.drivers.static.rules]
-"/home" = "localhost:19000"
-"123e4567-e89b-12d3-a456-426655440000" = "localhost:19000"
+[grpc.services.storageregistry.drivers.static.rules."/home"]
+address = "localhost:19000"
+[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"]
+address = "localhost:19000"
 
 [grpc.services.usershareprovider]
 driver = "memory"

diff --git a/examples/ocmd/ocmd-server-2.toml b/examples/ocmd/ocmd-server-2.toml
@@ -31,9 +31,10 @@ driver = "static"
 [grpc.services.storageregistry.drivers.static]
 home_provider = "/home"
 
-[grpc.services.storageregistry.drivers.static.rules]
-"/home" = "localhost:17000"
-"123e4567-e89b-12d3-a456-426655440000" = "localhost:17000"
+[grpc.services.storageregistry.drivers.static.rules."/home"]
+address = "localhost:17000"
+[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"]
+address = "localhost:17000"
 
 [grpc.services.usershareprovider]
 driver = "memory"

diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml
@@ -6,11 +6,12 @@ commit_share_to_storage_ref = true
 [grpc.services.storageregistry]
 [grpc.services.storageregistry.drivers.static]
 home_provider = "/home"
-
-[grpc.services.storageregistry.drivers.static.rules]
-"/home" = "localhost:17000"
-"/reva" = "localhost:18000"
-"123e4567-e89b-12d3-a456-426655440000" = "localhost:18000"
+[grpc.services.storageregistry.drivers.static.rules."/home"]
+address = "localhost:17000"
+[grpc.services.storageregistry.drivers.static.rules."/reva"]
+address = "localhost:18000"
+[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"]
+address = "localhost:18000"
 
 [grpc.services.authprovider]
 [grpc.services.authregistry]

diff --git a/go.mod b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e
-	github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c
+	github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59
 	github.com/ffurano/grpc-proto v0.0.0-20210312134900-65801a1ca184

diff --git a/go.sum b/go.sum
@@ -142,8 +142,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8=
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
-github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c h1:2vcWjiaFkJMhMZHeTbkkXWwhhAOTAIKpul8yjAo95UU=
-github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
+github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b h1:80DK9Yufaj1YJ0fPb6x1WZfijHWA+CMstq3MEZs/8To=
+github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
 github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go
@@ -55,6 +55,9 @@ func (s *svc) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareRequest
 	// TODO(labkode): if both commits are enabled they could be done concurrently.
 	if s.c.CommitShareToStorageGrant {
 		addGrantStatus, err := s.addGrant(ctx, req.ResourceId, req.Grant.Grantee, req.Grant.Permissions.Permissions)
+		if err != nil {
+			return nil, errors.Wrap(err, "gateway: error adding OCM grant to storage")
+		}
 		if addGrantStatus.Code != rpc.Code_CODE_OK {
 			return &ocm.CreateOCMShareResponse{
 				Status: addGrantStatus,
@@ -104,6 +107,9 @@ func (s *svc) RemoveOCMShare(ctx context.Context, req *ocm.RemoveOCMShareRequest
 	// TODO(labkode): if both commits are enabled they could be done concurrently.
 	if s.c.CommitShareToStorageGrant {
 		removeGrantStatus, err := s.removeGrant(ctx, share.ResourceId, share.Grantee, share.Permissions.Permissions)
+		if err != nil {
+			return nil, errors.Wrap(err, "gateway: error removing OCM grant from storage")
+		}
 		if removeGrantStatus.Code != rpc.Code_CODE_OK {
 			return &ocm.RemoveOCMShareResponse{
 				Status: removeGrantStatus,
Original file line number	Diff line number	Diff line change
Expand Up		@@ -35,4 +35,3 @@ linters:
		- gocritic
		- prealloc
		#- gosec