From 538c8bdafb0ceeca96f650e580885dfeb4035276 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 25 Mar 2021 15:38:42 +0100 Subject: [PATCH] Enhance storage registry with virtual views and regular expressions. (#1570) --- .golangci.yaml | 1 - .../unreleased/storage-registry-refactor.md | 9 + cmd/revad/main.go | 2 +- examples/oc-phoenix/gateway.toml | 12 +- examples/ocmd/ocmd-server-1.toml | 7 +- examples/ocmd/ocmd-server-2.toml | 7 +- examples/storage-references/gateway.toml | 11 +- go.mod | 2 +- go.sum | 4 +- .../grpc/services/gateway/ocmshareprovider.go | 6 + .../grpc/services/gateway/storageprovider.go | 210 ++++++++++--- .../services/gateway/usershareprovider.go | 3 + .../storageregistry/storageregistry.go | 26 +- .../services/owncloud/ocs/config/config.go | 5 + .../handlers/apps/sharing/shares/shares.go | 49 +--- pkg/cbox/utils/conversions.go | 20 +- pkg/cbox/utils/tokenmanagement.go | 4 +- pkg/rhttp/datatx/utils/download/download.go | 2 +- pkg/storage/registry/static/static.go | 161 +++++++--- .../registry/static/static_suite_test.go | 31 ++ pkg/storage/registry/static/static_test.go | 277 ++++++++++++++++++ pkg/storage/storage.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 5 +- pkg/storage/utils/etag/etag.go | 16 +- tests/oc-integration-tests/drone/gateway.toml | 13 +- tests/oc-integration-tests/local/gateway.toml | 13 +- 26 files changed, 696 insertions(+), 202 deletions(-) create mode 100644 changelog/unreleased/storage-registry-refactor.md create mode 100644 pkg/storage/registry/static/static_suite_test.go create mode 100644 pkg/storage/registry/static/static_test.go diff --git a/.golangci.yaml b/.golangci.yaml index 8354f33902..c99e544113 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -35,4 +35,3 @@ linters: - gocritic - prealloc #- gosec - diff --git a/changelog/unreleased/storage-registry-refactor.md b/changelog/unreleased/storage-registry-refactor.md new file mode 100644 index 0000000000..de9db60da5 --- /dev/null +++ b/changelog/unreleased/storage-registry-refactor.md @@ -0,0 +1,9 @@ +Enhancement: Enhance storage registry with virtual views and regular expressions + +Add the functionality to the storage registry service to handle user requests +for references which can span across multiple storage providers, particularly +useful for cases where directories are sharded across providers or virtual views +are expected. + +https://github.com/cs3org/cs3apis/pull/116 +https://github.com/cs3org/reva/pull/1570 diff --git a/cmd/revad/main.go b/cmd/revad/main.go index 4347bdaa7f..e803757c3d 100644 --- a/cmd/revad/main.go +++ b/cmd/revad/main.go @@ -227,8 +227,8 @@ func runMultiple(confs []map[string]interface{}) { wg.Add(1) pidfile := getPidfile() go func(wg *sync.WaitGroup, conf map[string]interface{}) { + defer wg.Done() runtime.Run(conf, pidfile, *logFlag) - wg.Done() }(&wg, conf) } wg.Wait() diff --git a/examples/oc-phoenix/gateway.toml b/examples/oc-phoenix/gateway.toml index 2ea1f23ec6..3ce3462226 100644 --- a/examples/oc-phoenix/gateway.toml +++ b/examples/oc-phoenix/gateway.toml @@ -48,15 +48,17 @@ driver = "static" [grpc.services.storageregistry.drivers.static] home_provider = "/home" -[grpc.services.storageregistry.drivers.static.rules] # mount a home storage provider that uses a context based path wrapper # to jail users into their home dir -"/home" = "localhost:12000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:12000" # mount a storage provider without a path wrapper for direct access to users. -"/oc" = "localhost:11000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:11000" -# another mount point might be "/projects/" +[grpc.services.storageregistry.drivers.static.rules."/oc"] +address = "localhost:11000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:11000" +# another mount point might be "/projects/" [http] address = "0.0.0.0:19001" diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index e65b8233dd..9d0335ced0 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -33,9 +33,10 @@ driver = "static" [grpc.services.storageregistry.drivers.static] home_provider = "/home" -[grpc.services.storageregistry.drivers.static.rules] -"/home" = "localhost:19000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:19000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:19000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:19000" [grpc.services.usershareprovider] driver = "memory" diff --git a/examples/ocmd/ocmd-server-2.toml b/examples/ocmd/ocmd-server-2.toml index 3255843e58..321cdcd1ce 100644 --- a/examples/ocmd/ocmd-server-2.toml +++ b/examples/ocmd/ocmd-server-2.toml @@ -31,9 +31,10 @@ driver = "static" [grpc.services.storageregistry.drivers.static] home_provider = "/home" -[grpc.services.storageregistry.drivers.static.rules] -"/home" = "localhost:17000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:17000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:17000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:17000" [grpc.services.usershareprovider] driver = "memory" diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 7a5359a2db..342d8fa17d 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -6,11 +6,12 @@ commit_share_to_storage_ref = true [grpc.services.storageregistry] [grpc.services.storageregistry.drivers.static] home_provider = "/home" - -[grpc.services.storageregistry.drivers.static.rules] -"/home" = "localhost:17000" -"/reva" = "localhost:18000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:18000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:17000" +[grpc.services.storageregistry.drivers.static.rules."/reva"] +address = "localhost:18000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:18000" [grpc.services.authprovider] [grpc.services.authregistry] diff --git a/go.mod b/go.mod index 4a0adc77e2..7300366f0b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c + github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/go-ldap/ldap/v3 v3.2.4 diff --git a/go.sum b/go.sum index ee8125224f..7ca92662a6 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c h1:2vcWjiaFkJMhMZHeTbkkXWwhhAOTAIKpul8yjAo95UU= -github.com/cs3org/go-cs3apis v0.0.0-20210316113645-e4a74cb8761c/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b h1:80DK9Yufaj1YJ0fPb6x1WZfijHWA+CMstq3MEZs/8To= +github.com/cs3org/go-cs3apis v0.0.0-20210322124405-872bbbf14d0b/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index c663f2b8dd..5b91eb9f07 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -55,6 +55,9 @@ func (s *svc) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareRequest // TODO(labkode): if both commits are enabled they could be done concurrently. if s.c.CommitShareToStorageGrant { addGrantStatus, err := s.addGrant(ctx, req.ResourceId, req.Grant.Grantee, req.Grant.Permissions.Permissions) + if err != nil { + return nil, errors.Wrap(err, "gateway: error adding OCM grant to storage") + } if addGrantStatus.Code != rpc.Code_CODE_OK { return &ocm.CreateOCMShareResponse{ Status: addGrantStatus, @@ -104,6 +107,9 @@ func (s *svc) RemoveOCMShare(ctx context.Context, req *ocm.RemoveOCMShareRequest // TODO(labkode): if both commits are enabled they could be done concurrently. if s.c.CommitShareToStorageGrant { removeGrantStatus, err := s.removeGrant(ctx, share.ResourceId, share.Grantee, share.Permissions.Permissions) + if err != nil { + return nil, errors.Wrap(err, "gateway: error removing OCM grant from storage") + } if removeGrantStatus.Code != rpc.Code_CODE_OK { return &ocm.RemoveOCMShareResponse{ Status: removeGrantStatus, diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 38b61eeab2..eba1cd1464 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -24,6 +24,7 @@ import ( "net/url" "path" "strings" + "sync" "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -35,10 +36,9 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/storage/utils/etag" - "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/utils" "github.com/dgrijalva/jwt-go" + "github.com/google/uuid" "github.com/pkg/errors" ) @@ -193,6 +193,7 @@ func (s *svc) getHome(_ context.Context) string { // TODO(labkode): issue #601, /home will be hardcoded. return "/home" } + func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) { log := appctx.GetLogger(ctx) p, st := s.getPath(ctx, req.Ref) @@ -366,6 +367,7 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi } func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) { + // TODO(ishank011): enable downloading references spread across storage providers, eg. /eos c, err := s.find(ctx, req.Ref) if err != nil { return &gateway.InitiateFileDownloadResponse{ @@ -857,6 +859,7 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide } func (s *svc) delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { + // TODO(ishank011): enable deleting references spread across storage providers, eg. /eos c, err := s.find(ctx, req.Ref) if err != nil { return &provider.DeleteResponse{ @@ -974,19 +977,20 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo } func (s *svc) move(ctx context.Context, req *provider.MoveRequest) (*provider.MoveResponse, error) { - srcP, err := s.findProvider(ctx, req.Source) + srcList, err := s.findProviders(ctx, req.Source) if err != nil { return &provider.MoveResponse{ Status: status.NewStatusFromErrType(ctx, "move src="+req.Source.String(), err), }, nil } - dstP, err := s.findProvider(ctx, req.Destination) + dstList, err := s.findProviders(ctx, req.Destination) if err != nil { return &provider.MoveResponse{ Status: status.NewStatusFromErrType(ctx, "move dst="+req.Destination.String(), err), }, nil } + srcP, dstP := srcList[0], dstList[0] // if providers are not the same we do not implement cross storage copy yet. if srcP.Address != dstP.Address { @@ -1007,6 +1011,7 @@ func (s *svc) move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo } func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { + // TODO(ishank011): enable for references spread across storage providers, eg. /eos c, err := s.find(ctx, req.Ref) if err != nil { return &provider.SetArbitraryMetadataResponse{ @@ -1023,6 +1028,7 @@ func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitra } func (s *svc) UnsetArbitraryMetadata(ctx context.Context, req *provider.UnsetArbitraryMetadataRequest) (*provider.UnsetArbitraryMetadataResponse, error) { + // TODO(ishank011): enable for references spread across storage providers, eg. /eos c, err := s.find(ctx, req.Ref) if err != nil { return &provider.UnsetArbitraryMetadataResponse{ @@ -1142,14 +1148,89 @@ func (s *svc) statSharesFolder(ctx context.Context) (*provider.StatResponse, err } func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) { - c, err := s.find(ctx, req.Ref) + providers, err := s.findProviders(ctx, req.Ref) if err != nil { return &provider.StatResponse{ - Status: status.NewStatusFromErrType(ctx, "stat ref="+req.Ref.String(), err), + Status: status.NewStatusFromErrType(ctx, "stat ref: "+req.Ref.String(), err), }, nil } - return c.Stat(ctx, req) + resPath := req.Ref.GetPath() + if len(providers) == 1 && (resPath == "" || strings.HasPrefix(resPath, providers[0].ProviderPath)) { + c, err := s.getStorageProviderClient(ctx, providers[0]) + if err != nil { + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "error connecting to storage provider="+providers[0].Address), + }, nil + } + return c.Stat(ctx, req) + } + + infoFromProviders := make([]*provider.ResourceInfo, len(providers)) + errors := make([]error, len(providers)) + var wg sync.WaitGroup + + for i, p := range providers { + wg.Add(1) + go s.statOnProvider(ctx, req, infoFromProviders[i], p, &errors[i], &wg) + } + wg.Wait() + + var totalSize uint64 + for i := range providers { + if errors[i] != nil { + return &provider.StatResponse{ + Status: status.NewStatusFromErrType(ctx, "stat ref: "+req.Ref.String(), errors[i]), + }, nil + } + if infoFromProviders[i] != nil { + totalSize += infoFromProviders[i].Size + } + } + + // TODO(ishank011): aggregrate other properties for references spread across storage providers, eg. /eos + return &provider.StatResponse{ + Status: status.NewOK(ctx), + Info: &provider.ResourceInfo{ + Id: &provider.ResourceId{ + StorageId: "/", + OpaqueId: uuid.New().String(), + }, + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Path: resPath, + Size: totalSize, + }, + }, nil +} + +func (s *svc) statOnProvider(ctx context.Context, req *provider.StatRequest, res *provider.ResourceInfo, p *registry.ProviderInfo, e *error, wg *sync.WaitGroup) { + defer wg.Done() + c, err := s.getStorageProviderClient(ctx, p) + if err != nil { + *e = errors.Wrap(err, "error connecting to storage provider="+p.Address) + return + } + + resPath := path.Clean(req.Ref.GetPath()) + newPath := req.Ref.GetPath() + if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) { + newPath = p.ProviderPath + } + r, err := c.Stat(ctx, &provider.StatRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: newPath, + }, + }, + }) + if err != nil { + *e = errors.Wrap(err, "gateway: error calling ListContainer") + return + } + if res == nil { + res = &provider.ResourceInfo{} + } + *res = *r.Info } func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) { @@ -1454,19 +1535,88 @@ func (s *svc) listSharesFolder(ctx context.Context) (*provider.ListContainerResp } func (s *svc) listContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) { - c, err := s.find(ctx, req.Ref) + providers, err := s.findProviders(ctx, req.Ref) if err != nil { return &provider.ListContainerResponse{ - Status: status.NewStatusFromErrType(ctx, "listContainer ref="+req.Ref.String(), err), + Status: status.NewStatusFromErrType(ctx, "listContainer ref: "+req.Ref.String(), err), }, nil } - res, err := c.ListContainer(ctx, req) + resPath := path.Clean(req.Ref.GetPath()) + infoFromProviders := make([][]*provider.ResourceInfo, len(providers)) + errors := make([]error, len(providers)) + var wg sync.WaitGroup + + for i, p := range providers { + wg.Add(1) + go s.listContainerOnProvider(ctx, req, &infoFromProviders[i], p, &errors[i], &wg) + } + wg.Wait() + + infos := []*provider.ResourceInfo{} + indirects := make(map[string][]*provider.ResourceInfo) + for i := range providers { + if errors[i] != nil { + return &provider.ListContainerResponse{ + Status: status.NewStatusFromErrType(ctx, "listContainer ref: "+req.Ref.String(), errors[i]), + }, nil + } + for _, inf := range infoFromProviders[i] { + if parent := path.Dir(inf.Path); resPath != "" && resPath != parent { + parts := strings.Split(strings.TrimPrefix(inf.Path, resPath), "/") + p := path.Join(resPath, parts[1]) + indirects[p] = append(indirects[p], inf) + } else { + infos = append(infos, inf) + } + } + } + + for k, v := range indirects { + inf := &provider.ResourceInfo{ + Id: &provider.ResourceId{ + StorageId: "/", + OpaqueId: uuid.New().String(), + }, + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Etag: etag.GenerateEtagFromResources(nil, v), + Path: k, + Size: 0, + } + infos = append(infos, inf) + } + + return &provider.ListContainerResponse{ + Status: status.NewOK(ctx), + Infos: infos, + }, nil +} + +func (s *svc) listContainerOnProvider(ctx context.Context, req *provider.ListContainerRequest, res *[]*provider.ResourceInfo, p *registry.ProviderInfo, e *error, wg *sync.WaitGroup) { + defer wg.Done() + c, err := s.getStorageProviderClient(ctx, p) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling ListContainer") + *e = errors.Wrap(err, "error connecting to storage provider="+p.Address) + return } - return res, nil + resPath := path.Clean(req.Ref.GetPath()) + newPath := req.Ref.GetPath() + if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) { + newPath = p.ProviderPath + } + r, err := c.ListContainer(ctx, &provider.ListContainerRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: newPath, + }, + }, + }) + if err != nil { + *e = errors.Wrap(err, "gateway: error calling ListContainer") + return + } + *res = r.Infos } func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) { @@ -1888,11 +2038,11 @@ func (s *svc) findByPath(ctx context.Context, path string) (provider.ProviderAPI } func (s *svc) find(ctx context.Context, ref *provider.Reference) (provider.ProviderAPIClient, error) { - p, err := s.findProvider(ctx, ref) + p, err := s.findProviders(ctx, ref) if err != nil { return nil, err } - return s.getStorageProviderClient(ctx, p) + return s.getStorageProviderClient(ctx, p[0]) } func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderInfo) (provider.ProviderAPIClient, error) { @@ -1905,37 +2055,13 @@ func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderIn return c, nil } -func (s *svc) findProvider(ctx context.Context, ref *provider.Reference) (*registry.ProviderInfo, error) { - home := s.getHome(ctx) - if strings.HasPrefix(ref.GetPath(), home) && s.c.HomeMapping != "" { - if u, ok := user.ContextGetUser(ctx); ok { - layout := templates.WithUser(u, s.c.HomeMapping) - newRef := &provider.Reference{ - Spec: &provider.Reference_Path{ - Path: path.Join(layout, strings.TrimPrefix(ref.GetPath(), home)), - }, - } - res, err := s.getStorageProvider(ctx, newRef) - if err != nil { - // if we get a NotFound error, default to the original reference - if _, ok := err.(errtypes.IsNotFound); !ok { - return nil, err - } - } else { - return res, nil - } - } - } - return s.getStorageProvider(ctx, ref) -} - -func (s *svc) getStorageProvider(ctx context.Context, ref *provider.Reference) (*registry.ProviderInfo, error) { +func (s *svc) findProviders(ctx context.Context, ref *provider.Reference) ([]*registry.ProviderInfo, error) { c, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint) if err != nil { return nil, errors.Wrap(err, "gateway: error getting storage registry client") } - res, err := c.GetStorageProvider(ctx, ®istry.GetStorageProviderRequest{ + res, err := c.GetStorageProviders(ctx, ®istry.GetStorageProvidersRequest{ Ref: ref, }) @@ -1958,11 +2084,11 @@ func (s *svc) getStorageProvider(ctx context.Context, ref *provider.Reference) ( } } - if res.Provider == nil { + if res.Providers == nil { return nil, errors.New("gateway: provider is nil") } - return res.Provider, nil + return res.Providers, nil } type etagWithTS struct { diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index f9d0395c37..1efe048526 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -119,6 +119,9 @@ func (s *svc) RemoveShare(ctx context.Context, req *collaboration.RemoveShareReq // TODO(labkode): if both commits are enabled they could be done concurrently. if s.c.CommitShareToStorageGrant { removeGrantStatus, err := s.removeGrant(ctx, share.ResourceId, share.Grantee, share.Permissions.Permissions) + if err != nil { + return nil, errors.Wrap(err, "gateway: error removing grant from storage") + } if removeGrantStatus.Code != rpc.Code_CODE_OK { return &collaboration.RemoveShareResponse{ Status: removeGrantStatus, diff --git a/internal/grpc/services/storageregistry/storageregistry.go b/internal/grpc/services/storageregistry/storageregistry.go index ba3e3bff67..02b6dbb5d8 100644 --- a/internal/grpc/services/storageregistry/storageregistry.go +++ b/internal/grpc/services/storageregistry/storageregistry.go @@ -108,38 +108,31 @@ func (s *service) ListStorageProviders(ctx context.Context, req *registrypb.List }, nil } - providers := make([]*registrypb.ProviderInfo, 0, len(pinfos)) - for _, info := range pinfos { - fill(info) - providers = append(providers, info) - } - res := ®istrypb.ListStorageProvidersResponse{ Status: status.NewOK(ctx), - Providers: providers, + Providers: pinfos, } return res, nil } -func (s *service) GetStorageProvider(ctx context.Context, req *registrypb.GetStorageProviderRequest) (*registrypb.GetStorageProviderResponse, error) { - p, err := s.reg.FindProvider(ctx, req.Ref) +func (s *service) GetStorageProviders(ctx context.Context, req *registrypb.GetStorageProvidersRequest) (*registrypb.GetStorageProvidersResponse, error) { + p, err := s.reg.FindProviders(ctx, req.Ref) if err != nil { switch err.(type) { case errtypes.IsNotFound: - return ®istrypb.GetStorageProviderResponse{ + return ®istrypb.GetStorageProvidersResponse{ Status: status.NewNotFound(ctx, err.Error()), }, nil default: - return ®istrypb.GetStorageProviderResponse{ + return ®istrypb.GetStorageProvidersResponse{ Status: status.NewInternal(ctx, err, "error finding storage provider"), }, nil } } - fill(p) - res := ®istrypb.GetStorageProviderResponse{ - Status: status.NewOK(ctx), - Provider: p, + res := ®istrypb.GetStorageProvidersResponse{ + Status: status.NewOK(ctx), + Providers: p, } return res, nil } @@ -161,6 +154,3 @@ func (s *service) GetHome(ctx context.Context, req *registrypb.GetHomeRequest) ( } return res, nil } - -// TODO(labkode): fix -func fill(p *registrypb.ProviderInfo) {} diff --git a/internal/http/services/owncloud/ocs/config/config.go b/internal/http/services/owncloud/ocs/config/config.go index 725be7eacb..3cb402e269 100644 --- a/internal/http/services/owncloud/ocs/config/config.go +++ b/internal/http/services/owncloud/ocs/config/config.go @@ -32,6 +32,7 @@ type Config struct { DefaultUploadProtocol string `mapstructure:"default_upload_protocol"` UserAgentChunkingMap map[string]string `mapstructure:"user_agent_chunking_map"` SharePrefix string `mapstructure:"share_prefix"` + HomeNamespace string `mapstructure:"home_namespace"` } // Init sets sane defaults @@ -48,5 +49,9 @@ func (c *Config) Init() { c.DefaultUploadProtocol = "tus" } + if c.HomeNamespace == "" { + c.HomeNamespace = "/home" + } + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index f4ddf401af..028c3e9172 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -55,6 +55,7 @@ type Handler struct { gatewayAddr string publicURL string sharePrefix string + homeNamespace string userIdentifierCache *ttlcache.Cache } @@ -70,6 +71,7 @@ func (h *Handler) Init(c *config.Config) error { h.gatewayAddr = c.GatewaySvc h.publicURL = c.Config.Host h.sharePrefix = c.SharePrefix + h.homeNamespace = c.HomeNamespace h.userIdentifierCache = ttlcache.NewCache() _ = h.userIdentifierCache.SetTTL(60 * time.Second) @@ -173,16 +175,9 @@ func (h *Handler) createShare(w http.ResponseWriter, r *http.Request) { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err) return } - // prefix the path with the owners home, because ocs share requests are relative to the home dir - // TODO the path actually depends on the configured webdav_namespace - hRes, err := c.GetHome(ctx, &provider.GetHomeRequest{}) - if err != nil { - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err) - return - } - prefix := hRes.GetPath() - fn := path.Join(prefix, r.FormValue("path")) + // prefix the path with the owners home, because ocs share requests are relative to the home dir + fn := path.Join(h.homeNamespace, r.FormValue("path")) statReq := provider.StatRequest{ Ref: &provider.Reference{ @@ -591,14 +586,7 @@ func (h *Handler) listSharesWithMe(w http.ResponseWriter, r *http.Request) { // we need to lookup the resource id so we can filter the list of shares later if p != "" { // prefix the path with the owners home, because ocs share requests are relative to the home dir - // TODO the path actually depends on the configured webdav_namespace - hRes, err := gwc.GetHome(ctx, &provider.GetHomeRequest{}) - if err != nil { - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err) - return - } - - target := path.Join(hRes.Path, r.FormValue("path")) + target := path.Join(h.homeNamespace, r.FormValue("path")) statReq := &provider.StatRequest{ Ref: &provider.Reference{ @@ -728,33 +716,8 @@ func (h *Handler) listSharesWithOthers(w http.ResponseWriter, r *http.Request) { // shared with others p := r.URL.Query().Get("path") if p != "" { - c, err := pool.GetGatewayServiceClient(h.gatewayAddr) - if err != nil { - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting storage grpc client", err) - return - } - // prefix the path with the owners home, because ocs share requests are relative to the home dir - // TODO the path actually depends on the configured webdav_namespace - hRes, err := c.GetHome(r.Context(), &provider.GetHomeRequest{}) - if err != nil { - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err) - return - } - - if hRes.Status.Code != rpc.Code_CODE_OK { - switch hRes.Status.Code { - case rpc.Code_CODE_NOT_FOUND: - response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "path not found", nil) - case rpc.Code_CODE_PERMISSION_DENIED: - response.WriteOCSError(w, r, response.MetaUnauthorized.StatusCode, "permission denied", nil) - default: - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc stat request failed", nil) - } - return - } - - filters, linkFilters, err = h.addFilters(w, r, hRes.GetPath()) + filters, linkFilters, err = h.addFilters(w, r, h.homeNamespace) if err != nil { // result has been written as part of addFilters return diff --git a/pkg/cbox/utils/conversions.go b/pkg/cbox/utils/conversions.go index 1997c66664..784debffbf 100644 --- a/pkg/cbox/utils/conversions.go +++ b/pkg/cbox/utils/conversions.go @@ -19,8 +19,6 @@ package utils import ( - "fmt" - "strings" "time" grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" @@ -161,36 +159,22 @@ func IntToShareState(g int) collaboration.ShareState { // FormatUserID formats a CS3API user ID to a string func FormatUserID(u *userpb.UserId) string { - if u.Idp != "" { - return fmt.Sprintf("%s:%s", u.OpaqueId, u.Idp) - } return u.OpaqueId } // ExtractUserID retrieves a CS3API user ID from a string func ExtractUserID(u string) *userpb.UserId { - parts := strings.SplitN(u, ":", 2) - if len(parts) > 1 { - return &userpb.UserId{OpaqueId: parts[0], Idp: parts[1]} - } - return &userpb.UserId{OpaqueId: parts[0]} + return &userpb.UserId{OpaqueId: u} } // FormatGroupID formats a CS3API group ID to a string func FormatGroupID(u *grouppb.GroupId) string { - if u.Idp != "" { - return fmt.Sprintf("%s:%s", u.OpaqueId, u.Idp) - } return u.OpaqueId } // ExtractGroupID retrieves a CS3API group ID from a string func ExtractGroupID(u string) *grouppb.GroupId { - parts := strings.SplitN(u, ":", 2) - if len(parts) > 1 { - return &grouppb.GroupId{OpaqueId: parts[0], Idp: parts[1]} - } - return &grouppb.GroupId{OpaqueId: parts[0]} + return &grouppb.GroupId{OpaqueId: u} } // ConvertToCS3Share converts a DBShare to a CS3API collaboration share diff --git a/pkg/cbox/utils/tokenmanagement.go b/pkg/cbox/utils/tokenmanagement.go index 66600dc3f8..e16fd503ba 100644 --- a/pkg/cbox/utils/tokenmanagement.go +++ b/pkg/cbox/utils/tokenmanagement.go @@ -90,8 +90,8 @@ func (a *APITokenManager) renewAPIToken(ctx context.Context, forceRenewal bool) func (a *APITokenManager) getAPIToken(ctx context.Context) (string, time.Time, error) { params := url.Values{ - "grant_types": {"client_credentials"}, - "audience": {a.conf.TargetAPI}, + "grant_type": {"client_credentials"}, + "audience": {a.conf.TargetAPI}, } httpReq, err := http.NewRequest("POST", a.conf.OIDCTokenEndpoint, strings.NewReader(params.Encode())) diff --git a/pkg/rhttp/datatx/utils/download/download.go b/pkg/rhttp/datatx/utils/download/download.go index 724fbd1649..f7de5fab9c 100644 --- a/pkg/rhttp/datatx/utils/download/download.go +++ b/pkg/rhttp/datatx/utils/download/download.go @@ -95,9 +95,9 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS) { var s io.Seeker if s, ok = content.(io.Seeker); ok { // tell clients they can send range requests - w.Header().Set("Accept-Ranges", "bytes") } + if len(ranges) > 0 { sublog.Debug().Int64("start", ranges[0].Start).Int64("length", ranges[0].Length).Msg("range request") if s == nil { diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index 0916b2e1aa..0242011ecf 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -20,6 +20,8 @@ package static import ( "context" + "path" + "regexp" "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -28,6 +30,8 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/registry/registry" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -36,9 +40,17 @@ func init() { registry.Register("static", New) } +var bracketRegex = regexp.MustCompile(`\[(.*?)\]`) + +type rule struct { + Mapping string `mapstructure:"mapping"` + Address string `mapstructure:"address"` + Aliases map[string]string `mapstructure:"aliases"` +} + type config struct { - Rules map[string]string `mapstructure:"rules"` - HomeProvider string `mapstructure:"home_provider"` + Rules map[string]rule `mapstructure:"rules"` + HomeProvider string `mapstructure:"home_provider"` } func (c *config) init() { @@ -47,9 +59,13 @@ func (c *config) init() { } if len(c.Rules) == 0 { - c.Rules = map[string]string{ - "/": sharedconf.GetGatewaySVC(""), - "00000000-0000-0000-0000-000000000000": sharedconf.GetGatewaySVC(""), + c.Rules = map[string]rule{ + "/": rule{ + Address: sharedconf.GetGatewaySVC(""), + }, + "00000000-0000-0000-0000-000000000000": rule{ + Address: sharedconf.GetGatewaySVC(""), + }, } } } @@ -62,8 +78,8 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. +// New returns an implementation of the storage.Registry interface that +// redirects requests to corresponding storage drivers. func New(m map[string]interface{}) (storage.Registry, error) { c, err := parseConfig(m) if err != nil { @@ -77,63 +93,134 @@ type reg struct { c *config } +func getProviderAddr(ctx context.Context, r rule) string { + addr := r.Address + if addr == "" { + if u, ok := user.ContextGetUser(ctx); ok { + layout := templates.WithUser(u, r.Mapping) + for k, v := range r.Aliases { + if match, _ := regexp.MatchString("^"+k, layout); match { + addr = v + } + } + } + } + return addr +} + func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { providers := []*registrypb.ProviderInfo{} for k, v := range b.c.Rules { - providers = append(providers, ®istrypb.ProviderInfo{ - Address: v, - ProviderPath: k, - }) + if addr := getProviderAddr(ctx, v); addr != "" { + combs := generateRegexCombinations(k) + for _, c := range combs { + providers = append(providers, ®istrypb.ProviderInfo{ + ProviderPath: c, + Address: addr, + }) + } + } } return providers, nil } // returns the the root path of the first provider in the list. -// TODO(labkode): this is not production ready. func (b *reg) GetHome(ctx context.Context) (*registrypb.ProviderInfo, error) { - address, ok := b.c.Rules[b.c.HomeProvider] - if ok { - return ®istrypb.ProviderInfo{ - ProviderPath: b.c.HomeProvider, - Address: address, - }, nil + // Assume that HomeProvider is not a regexp + if r, ok := b.c.Rules[b.c.HomeProvider]; ok { + if addr := getProviderAddr(ctx, r); addr != "" { + return ®istrypb.ProviderInfo{ + ProviderPath: b.c.HomeProvider, + Address: addr, + }, nil + } } return nil, errors.New("static: home not found") } -func (b *reg) FindProvider(ctx context.Context, ref *provider.Reference) (*registrypb.ProviderInfo, error) { +func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*registrypb.ProviderInfo, error) { // find longest match - var match string + var match *registrypb.ProviderInfo + var shardedMatches []*registrypb.ProviderInfo - // we try to find first by path as most storage operations will be done on path. - fn := ref.GetPath() + // Try to find by path first as most storage operations will be done using the path. + fn := path.Clean(ref.GetPath()) if fn != "" { - for prefix := range b.c.Rules { - if strings.HasPrefix(fn, prefix) && len(prefix) > len(match) { - match = prefix + for prefix, rule := range b.c.Rules { + addr := getProviderAddr(ctx, rule) + r, err := regexp.Compile("^" + prefix) + if err != nil { + continue + } + if m := r.FindString(fn); m != "" { + match = ®istrypb.ProviderInfo{ + ProviderPath: m, + Address: addr, + } + } + // Check if the current rule forms a part of a reference spread across storage providers. + if strings.HasPrefix(prefix, fn) { + combs := generateRegexCombinations(prefix) + for _, c := range combs { + shardedMatches = append(shardedMatches, ®istrypb.ProviderInfo{ + ProviderPath: c, + Address: addr, + }) + } } } } - if match != "" { - return ®istrypb.ProviderInfo{ - ProviderPath: match, - Address: b.c.Rules[match], - }, nil + if match != nil && match.ProviderPath != "" { + return []*registrypb.ProviderInfo{match}, nil + } else if len(shardedMatches) > 0 { + // If we don't find a perfect match but at least one provider is encapsulated + // by the reference, return all such providers. + return shardedMatches, nil } - // we try with id + // Try with id id := ref.GetId() if id == nil { return nil, errtypes.NotFound("storage provider not found for ref " + ref.String()) } - address, ok := b.c.Rules[id.StorageId] - if ok { + + for prefix, rule := range b.c.Rules { + addr := getProviderAddr(ctx, rule) + r, err := regexp.Compile("^" + prefix + "$") + if err != nil { + continue + } // TODO(labkode): fill path info based on provider id, if path and storage id points to same id, take that. - return ®istrypb.ProviderInfo{ - ProviderId: id.StorageId, - Address: address, - }, nil + if m := r.FindString(id.StorageId); m != "" { + return []*registrypb.ProviderInfo{®istrypb.ProviderInfo{ + ProviderId: id.StorageId, + Address: addr, + }}, nil + } } + return nil, errtypes.NotFound("storage provider not found for ref " + ref.String()) } + +func generateRegexCombinations(rex string) []string { + m := bracketRegex.FindString(rex) + r := strings.Trim(strings.Trim(m, "["), "]") + if r == "" { + return []string{rex} + } + var combinations []string + for i := 0; i < len(r); i++ { + if i < len(r)-2 && r[i+1] == '-' { + for j := r[i]; j <= r[i+2]; j++ { + p := strings.Replace(rex, m, string(j), 1) + combinations = append(combinations, generateRegexCombinations(p)...) + } + i += 2 + } else { + p := strings.Replace(rex, m, string(r[i]), 1) + combinations = append(combinations, generateRegexCombinations(p)...) + } + } + return combinations +} diff --git a/pkg/storage/registry/static/static_suite_test.go b/pkg/storage/registry/static/static_suite_test.go new file mode 100644 index 0000000000..686ccf3360 --- /dev/null +++ b/pkg/storage/registry/static/static_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package static_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestStaticDriver(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Static driver suite") +} diff --git a/pkg/storage/registry/static/static_test.go b/pkg/storage/registry/static/static_test.go new file mode 100644 index 0000000000..ff0aee14cc --- /dev/null +++ b/pkg/storage/registry/static/static_test.go @@ -0,0 +1,277 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package static_test + +import ( + "context" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" + "github.com/cs3org/reva/pkg/storage/registry/static" + "github.com/cs3org/reva/pkg/user" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Static", func() { + + totalProviders, rootProviders, eosProviders := 32, 30, 28 + + handler, err := static.New(map[string]interface{}{ + "home_provider": "/home", + "rules": map[string]interface{}{ + "/home": map[string]interface{}{ + "mapping": "/home-{{substr 0 1 .Id.OpaqueId}}", + "aliases": map[string]string{ + "/home-[a-fg-o]": "home-00-home", + "/home-[pqrstu]": "home-01-home", + "/home-[v-z]": "home-02-home", + }, + }, + "/MyShares": map[string]interface{}{ + "mapping": "/MyShares-{{substr 0 1 .Id.OpaqueId}}", + "aliases": map[string]string{ + "/MyShares-[a-fg-o]": "home-00-shares", + "/MyShares-[pqrstu]": "home-01-shares", + "/MyShares-[v-z]": "home-02-shares", + }, + }, + "/eos/user/[a-fg-o]": map[string]interface{}{ + "address": "home-00-eos", + }, + "/eos/user/[pqrstu]": map[string]interface{}{ + "address": "home-01-eos", + }, + "/eos/user/[v-z]": map[string]interface{}{ + "address": "home-02-eos", + }, + "/eos/project": map[string]interface{}{ + "address": "project-00", + }, + "/eos/media": map[string]interface{}{ + "address": "media-00", + }, + "123e4567-e89b-12d3-a456-426655440000": map[string]interface{}{ + "address": "home-00-home", + }, + "123e4567-e89b-12d3-a456-426655440001": map[string]interface{}{ + "address": "home-01-home", + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + ctxAlice := user.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{ + OpaqueId: "alice", + }, + }) + ctxRobert := user.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{ + OpaqueId: "robert", + }, + }) + + Describe("ListProviders", func() { + It("lists all providers for user alice", func() { + providers, err := handler.ListProviders(ctxAlice) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(totalProviders)) + }) + + It("lists all providers for user robert", func() { + providers, err := handler.ListProviders(ctxRobert) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(totalProviders)) + }) + }) + + Describe("GetHome", func() { + It("get the home provider for user alice", func() { + home, err := handler.GetHome(ctxAlice) + Expect(err).ToNot(HaveOccurred()) + Expect(home).To(Equal(®istrypb.ProviderInfo{ + ProviderPath: "/home", + Address: "home-00-home", + })) + }) + + It("get the home provider for user robert", func() { + home, err := handler.GetHome(ctxRobert) + Expect(err).ToNot(HaveOccurred()) + Expect(home).To(Equal(®istrypb.ProviderInfo{ + ProviderPath: "/home", + Address: "home-01-home", + })) + }) + }) + + Describe("FindProviders for home reference", func() { + ref := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "/home/abcd", + }, + } + + It("finds all providers for user alice for a home ref", func() { + providers, err := handler.FindProviders(ctxAlice, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/home", + Address: "home-00-home", + }})) + }) + + It("finds all providers for user robert for a home ref", func() { + providers, err := handler.FindProviders(ctxRobert, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/home", + Address: "home-01-home", + }})) + }) + }) + + Describe("FindProviders for eos reference", func() { + ref := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "/eos/user/b/bob/xyz", + }, + } + + It("finds all providers for user alice for an eos ref", func() { + providers, err := handler.FindProviders(ctxAlice, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/eos/user/b", + Address: "home-00-eos", + }})) + }) + + It("finds all providers for user robert for an eos ref", func() { + providers, err := handler.FindProviders(ctxRobert, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/eos/user/b", + Address: "home-00-eos", + }})) + }) + }) + + Describe("FindProviders for project reference", func() { + ref := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "/eos/project/pqr", + }, + } + + It("finds all providers for user alice for a project ref", func() { + providers, err := handler.FindProviders(ctxAlice, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/eos/project", + Address: "project-00", + }})) + }) + + It("finds all providers for user robert for a project ref", func() { + providers, err := handler.FindProviders(ctxRobert, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderPath: "/eos/project", + Address: "project-00", + }})) + }) + }) + + Describe("FindProviders for virtual references", func() { + ref1 := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "/eos", + }, + } + ref2 := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "/", + }, + } + + It("finds all providers for user alice for a virtual eos ref", func() { + providers, err := handler.FindProviders(ctxAlice, ref1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(eosProviders)) + }) + + It("finds all providers for user robert for a virtual eos ref", func() { + providers, err := handler.FindProviders(ctxRobert, ref1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(eosProviders)) + }) + + It("finds all providers for user alice for a virtual root ref", func() { + providers, err := handler.FindProviders(ctxAlice, ref2) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(rootProviders)) + }) + + It("finds all providers for user robert for a virtual root ref", func() { + providers, err := handler.FindProviders(ctxRobert, ref2) + Expect(err).ToNot(HaveOccurred()) + Expect(len(providers)).To(Equal(rootProviders)) + }) + }) + + Describe("FindProviders for reference containing ID", func() { + ref := &provider.Reference{ + Spec: &provider.Reference_Id{ + Id: &provider.ResourceId{ + StorageId: "123e4567-e89b-12d3-a456-426655440000", + }, + }, + } + + It("finds all providers for user alice for ref containing ID", func() { + providers, err := handler.FindProviders(ctxAlice, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderId: "123e4567-e89b-12d3-a456-426655440000", + Address: "home-00-home", + }})) + }) + + It("finds all providers for user robert for ref containing ID", func() { + providers, err := handler.FindProviders(ctxRobert, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(providers).To(Equal([]*registrypb.ProviderInfo{ + ®istrypb.ProviderInfo{ + ProviderId: "123e4567-e89b-12d3-a456-426655440000", + Address: "home-00-home", + }})) + }) + }) +}) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a986eb4b18..d80d5fdb8e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -61,7 +61,7 @@ type FS interface { // Registry is the interface that storage registries implement // for discovering storage providers type Registry interface { - FindProvider(ctx context.Context, ref *provider.Reference) (*registry.ProviderInfo, error) + FindProviders(ctx context.Context, ref *provider.Reference) ([]*registry.ProviderInfo, error) ListProviders(ctx context.Context) ([]*registry.ProviderInfo, error) GetHome(ctx context.Context) (*registry.ProviderInfo, error) } diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 722bb9be04..c6956ea065 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -659,7 +659,8 @@ func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*p } } - if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil { + // Remove the hidden folders in the topmost directory + if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil && finfo.Path != "/" && !strings.HasPrefix(finfo.Path, "/.") { finfos = append(finfos, finfo) } } @@ -713,7 +714,7 @@ func (fs *eosfs) listHome(ctx context.Context, home string) ([]*provider.Resourc } } - if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil { + if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil && finfo.Path != "/" && !strings.HasPrefix(finfo.Path, "/.") { finfos = append(finfos, finfo) } } diff --git a/pkg/storage/utils/etag/etag.go b/pkg/storage/utils/etag/etag.go index 190bbd1521..eaa8662503 100644 --- a/pkg/storage/utils/etag/etag.go +++ b/pkg/storage/utils/etag/etag.go @@ -51,15 +51,17 @@ var ( // GenerateEtagFromResources creates a unique etag for the root folder deriving // information from its multiple children func GenerateEtagFromResources(root *provider.ResourceInfo, children []*provider.ResourceInfo) string { - if params := getEtagParams(eosMtimeEtag, root.Etag); len(params) > 0 { - mtime := time.Unix(int64(root.Mtime.Seconds), int64(root.Mtime.Nanos)) - for _, r := range children { - m := time.Unix(int64(r.Mtime.Seconds), int64(r.Mtime.Nanos)) - if m.After(mtime) { - mtime = m + if root != nil { + if params := getEtagParams(eosMtimeEtag, root.Etag); len(params) > 0 { + mtime := time.Unix(int64(root.Mtime.Seconds), int64(root.Mtime.Nanos)) + for _, r := range children { + m := time.Unix(int64(r.Mtime.Seconds), int64(r.Mtime.Nanos)) + if m.After(mtime) { + mtime = m + } } + return fmt.Sprintf("\"%s:%d.%s\"", params["inode"], mtime.Unix(), strconv.FormatInt(mtime.UnixNano(), 10)[:3]) } - return fmt.Sprintf("\"%s:%d.%s\"", params["inode"], mtime.Unix(), strconv.FormatInt(mtime.UnixNano(), 10)[:3]) } return combineEtags(children) diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index cd698955a6..a535422b3e 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -53,17 +53,20 @@ driver = "static" [grpc.services.storageregistry.drivers.static] home_provider = "/home" -[grpc.services.storageregistry.drivers.static.rules] # mount a home storage provider that uses a context based path wrapper # to jail users into their home dir -"/home" = "localhost:12000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:12000" # mount a storage provider without a path wrapper for direct access to users. -"/oc" = "localhost:11000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:11000" +[grpc.services.storageregistry.drivers.static.rules."/oc"] +address = "localhost:11000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:11000" # another mount point might be "/projects/" -"/public" = "localhost:13000" +[grpc.services.storageregistry.drivers.static.rules."/public"] +address = "localhost:13000" [http] address = "0.0.0.0:19001" diff --git a/tests/oc-integration-tests/local/gateway.toml b/tests/oc-integration-tests/local/gateway.toml index 3805824a2b..cfe0c34182 100644 --- a/tests/oc-integration-tests/local/gateway.toml +++ b/tests/oc-integration-tests/local/gateway.toml @@ -53,17 +53,20 @@ driver = "static" [grpc.services.storageregistry.drivers.static] home_provider = "/home" -[grpc.services.storageregistry.drivers.static.rules] # mount a home storage provider that uses a context based path wrapper # to jail users into their home dir -"/home" = "localhost:12000" +[grpc.services.storageregistry.drivers.static.rules."/home"] +address = "localhost:12000" # mount a storage provider without a path wrapper for direct access to users. -"/users" = "localhost:11000" -"123e4567-e89b-12d3-a456-426655440000" = "localhost:11000" +[grpc.services.storageregistry.drivers.static.rules."/oc"] +address = "localhost:11000" +[grpc.services.storageregistry.drivers.static.rules."123e4567-e89b-12d3-a456-426655440000"] +address = "localhost:11000" # another mount point might be "/projects/" -"/public" = "localhost:13000" +[grpc.services.storageregistry.drivers.static.rules."/public"] +address = "localhost:13000" [http] address = "0.0.0.0:19001"