Skip to content

Commit

Permalink
Add logic to ls and stat to process arbitrary metadata keys (#905)
Browse files Browse the repository at this point in the history 
* Add logic to ls and stat to process arbitrary metadata keys

* Modify getPath to a variadic function
  • Loading branch information
@ishank011
ishank011 authored Jun 30, 2020
1 parent 9bbdbee commit 419de24
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 74 deletions.
12 changes: 6 additions & 6 deletions internal/grpc/services/gateway/storageprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,7 @@ func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.St
}

func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) {
p, err := s.getPath(ctx, req.Ref)
p, err := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
if err != nil {
return &provider.StatResponse{
Status: status.NewInternal(ctx, err, "gateway: error getting path for ref"),
Expand Down Expand Up @@ -1054,7 +1054,7 @@ func (s *svc) listContainer(ctx context.Context, req *provider.ListContainerRequ
}

func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) {
p, err := s.getPath(ctx, req.Ref)
p, err := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
if err != nil {
return &provider.ListContainerResponse{
Status: status.NewInternal(ctx, err, "gateway: error getting path for ref"),
Expand Down Expand Up @@ -1140,7 +1140,7 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ
},
}

newReq := &provider.ListContainerRequest{Ref: ref}
newReq := &provider.ListContainerRequest{Ref: ref, ArbitraryMetadataKeys: req.ArbitraryMetadataKeys}
newRes, err := s.listContainer(ctx, newReq)
if err != nil {
return &provider.ListContainerResponse{
Expand Down Expand Up @@ -1213,7 +1213,7 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ
},
}

newReq := &provider.ListContainerRequest{Ref: ref}
newReq := &provider.ListContainerRequest{Ref: ref, ArbitraryMetadataKeys: req.ArbitraryMetadataKeys}
newRes, err := s.listContainer(ctx, newReq)
if err != nil {
return &provider.ListContainerResponse{
Expand Down Expand Up @@ -1242,13 +1242,13 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ
panic("gateway: stating an unknown path:" + p)
}

func (s *svc) getPath(ctx context.Context, ref *provider.Reference) (string, error) {
func (s *svc) getPath(ctx context.Context, ref *provider.Reference, keys ...string) (string, error) {
if ref.GetPath() != "" {
return ref.GetPath(), nil
}

if ref.GetId() != nil && ref.GetId().GetOpaqueId() != "" {
req := &provider.StatRequest{Ref: ref}
req := &provider.StatRequest{Ref: ref, ArbitraryMetadataKeys: keys}
res, err := s.stat(ctx, req)
if err != nil {
err = errors.Wrap(err, "gateway: error stating ref:"+ref.String())
Expand Down
8 changes: 4 additions & 4 deletions internal/grpc/services/storageprovider/storageprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,7 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide
}, nil
}

md, err := s.storage.GetMD(ctx, newRef)
md, err := s.storage.GetMD(ctx, newRef, req.ArbitraryMetadataKeys)
if err != nil {
var st *rpc.Status
if _, ok := err.(errtypes.IsNotFound); ok {
Expand Down Expand Up @@ -491,7 +491,7 @@ func (s *service) ListContainerStream(req *provider.ListContainerStreamRequest,
return nil
}

mds, err := s.storage.ListFolder(ctx, newRef)
mds, err := s.storage.ListFolder(ctx, newRef, req.ArbitraryMetadataKeys)
if err != nil {
res := &provider.ListContainerStreamResponse{
Status: status.NewInternal(ctx, err, "error listing folder"),
Expand Down Expand Up @@ -535,7 +535,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
}, nil
}

mds, err := s.storage.ListFolder(ctx, newRef)
mds, err := s.storage.ListFolder(ctx, newRef, req.ArbitraryMetadataKeys)
if err != nil {
return &provider.ListContainerResponse{
Status: status.NewInternal(ctx, err, "error listing folder"),
Expand Down Expand Up @@ -832,7 +832,7 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provide
idRef := &provider.Reference{
Spec: &provider.Reference_Id{
Id: &provider.ResourceId{
StorageId: "", // on purpose, we are unwrapping, bottom layers only need OpaqueId.
StorageId: "", // we are unwrapping on purpose, bottom layers only need OpaqueId.
OpaqueId: ref.GetId().OpaqueId,
},
},
Expand Down
14 changes: 7 additions & 7 deletions pkg/storage/fs/eosgrpc/eosgrpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ func (fs *eosfs) getGrantPermissionSet(mode string) *provider.ResourcePermission
return p
}

func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
u, err := getUser(ctx)
if err != nil {
return nil, err
Expand All @@ -643,7 +643,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.
// if path is home we need to add in the response any shadow folder in the shadow homedirectory.
if fs.conf.EnableHome {
if fs.isShareFolder(ctx, p) {
return fs.getMDShareFolder(ctx, p)
return fs.getMDShareFolder(ctx, p, mdKeys)
}
}

Expand All @@ -658,7 +658,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.
return fi, nil
}

func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.ResourceInfo, error) {
func (fs *eosfs) getMDShareFolder(ctx context.Context, p string, mdKeys []string) (*provider.ResourceInfo, error) {
u, err := getUser(ctx)
if err != nil {
return nil, err
Expand All @@ -677,7 +677,7 @@ func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.Reso
return fs.convertToFileReference(ctx, eosFileInfo), nil
}

func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) {
func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
u, err := getUser(ctx)
if err != nil {
return nil, errors.Wrap(err, "eos: no user in ctx")
Expand Down Expand Up @@ -1440,7 +1440,7 @@ func (fs *eosfs) getEosMetadata(finfo *eosclientgrpc.FileInfo) []byte {
No - CreateDir(ctx context.Context, fn string) error
No -Delete(ctx context.Context, ref *provider.Reference) error
No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error
No -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
No -GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error)
Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error)
No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error
No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error)
Expand Down Expand Up @@ -1471,7 +1471,7 @@ func (fs *eosfs) getEosMetadata(finfo *eosclientgrpc.FileInfo) []byte {
No - CreateDir(ctx context.Context, fn string) error
Maybe -Delete(ctx context.Context, ref *provider.Reference) error
No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error
Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
Yes -GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error)
Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error)
No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error
No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error)
Expand Down Expand Up @@ -1502,7 +1502,7 @@ func (fs *eosfs) getEosMetadata(finfo *eosclientgrpc.FileInfo) []byte {
No - CreateDir(ctx context.Context, fn string) error
Maybe -Delete(ctx context.Context, ref *provider.Reference) error
Yes -Move(ctx context.Context, oldRef, newRef *provider.Reference) error
Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
Yes -GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error)
No -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error)
No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error
No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error)
Expand Down
64 changes: 40 additions & 24 deletions pkg/storage/fs/owncloud/owncloud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ func (fs *ocfs) getOwner(internal string) string {
return ""
}

func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np string, c redis.Conn) *provider.ResourceInfo {
func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np string, c redis.Conn, mdKeys []string) *provider.ResourceInfo {
id := readOrCreateID(ctx, np, c)
fn := fs.unwrap(ctx, path.Join("/", np))

Expand All @@ -417,28 +417,42 @@ func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np st
etag = string(val)
}

// TODO how do we tell the storage provider/driver which arbitrary metadata to retrieve? an analogy to webdav allprops or a list of requested properties
favorite := ""
if u, ok := user.ContextGetUser(ctx); ok {
// the favorite flag is specific to the user, so we need to incorporate the userid
if uid := u.GetId(); uid != nil {
fa := fmt.Sprintf("%s%s@%s", favPrefix, uid.GetOpaqueId(), uid.GetIdp())
if val, err := xattr.Get(np, fa); err == nil {
appctx.GetLogger(ctx).Debug().
Str("np", np).
Str("favorite", string(val)).
Str("username", u.GetUsername()).
Msg("found favorite flag")
favorite = string(val)
mdKeysMap := make(map[string]struct{})
for _, k := range mdKeys {
mdKeysMap[k] = struct{}{}
}

var returnAllKeys bool
if _, ok := mdKeysMap["*"]; len(mdKeys) == 0 || ok {
returnAllKeys = true
}

metadata := map[string]string{}

favoriteKey := "http://owncloud.org/ns/favorite"
if _, ok := mdKeysMap[favoriteKey]; returnAllKeys || ok {
favorite := ""
if u, ok := user.ContextGetUser(ctx); ok {
// the favorite flag is specific to the user, so we need to incorporate the userid
if uid := u.GetId(); uid != nil {
fa := fmt.Sprintf("%s%s@%s", favPrefix, uid.GetOpaqueId(), uid.GetIdp())
if val, err := xattr.Get(np, fa); err == nil {
appctx.GetLogger(ctx).Debug().
Str("np", np).
Str("favorite", string(val)).
Str("username", u.GetUsername()).
Msg("found favorite flag")
favorite = string(val)
}
} else {
appctx.GetLogger(ctx).Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id")
}
} else {
appctx.GetLogger(ctx).Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id")
appctx.GetLogger(ctx).Error().Err(errtypes.UserRequired("userrequired")).Msg("error getting user from ctx")
}
} else {
appctx.GetLogger(ctx).Error().Err(errtypes.UserRequired("userrequired")).Msg("error getting user from ctx")
metadata[favoriteKey] = favorite
}

metadata := map[string]string{}
list, err := xattr.List(np)
if err == nil {
for _, entry := range list {
Expand All @@ -447,7 +461,10 @@ func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np st
continue
}
if val, err := xattr.Get(np, entry); err == nil {
metadata[entry[len(mdPrefix):]] = string(val)
k := entry[len(mdPrefix):]
if _, ok := mdKeysMap[k]; returnAllKeys || ok {
metadata[k] = string(val)
}
} else {
appctx.GetLogger(ctx).Error().Err(err).
Str("entry", entry).
Expand All @@ -458,7 +475,6 @@ func (fs *ocfs) convertToResourceInfo(ctx context.Context, fi os.FileInfo, np st
appctx.GetLogger(ctx).Error().Err(err).Msg("error getting list of extended attributes")
}

metadata["http://owncloud.org/ns/favorite"] = favorite
return &provider.ResourceInfo{
Id: &provider.ResourceId{OpaqueId: id},
Path: fn,
Expand Down Expand Up @@ -1293,7 +1309,7 @@ func (fs *ocfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (e
return nil
}

func (fs *ocfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
func (fs *ocfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
np, err := fs.resolve(ctx, ref)
if err != nil {
return nil, errors.Wrap(err, "ocfs: error resolving reference")
Expand All @@ -1308,12 +1324,12 @@ func (fs *ocfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.R
}
c := fs.pool.Get()
defer c.Close()
m := fs.convertToResourceInfo(ctx, md, np, c)
m := fs.convertToResourceInfo(ctx, md, np, c, mdKeys)

return m, nil
}

func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) {
func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
np, err := fs.resolve(ctx, ref)
if err != nil {
return nil, errors.Wrap(err, "ocfs: error resolving reference")
Expand All @@ -1333,7 +1349,7 @@ func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*pro
defer c.Close()
for i := range mds {
p := path.Join(np, mds[i].Name())
m := fs.convertToResourceInfo(ctx, mds[i], p, c)
m := fs.convertToResourceInfo(ctx, mds[i], p, c, mdKeys)
finfos = append(finfos, m)
}
return finfos, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/storage/fs/s3/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ func (fs *s3FS) Move(ctx context.Context, oldRef, newRef *provider.Reference) er
return nil
}

func (fs *s3FS) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
func (fs *s3FS) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
log := appctx.GetLogger(ctx)

fn, err := fs.resolve(ctx, ref)
Expand Down Expand Up @@ -511,7 +511,7 @@ func (fs *s3FS) GetMD(ctx context.Context, ref *provider.Reference) (*provider.R
return fs.normalizeHead(ctx, output, fn), nil
}

func (fs *s3FS) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) {
func (fs *s3FS) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
fn, err := fs.resolve(ctx, ref)
if err != nil {
return nil, errors.Wrap(err, "error resolving ref")
Expand Down
4 changes: 2 additions & 2 deletions pkg/storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ type FS interface {
CreateDir(ctx context.Context, fn string) error
Delete(ctx context.Context, ref *provider.Reference) error
Move(ctx context.Context, oldRef, newRef *provider.Reference) error
GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error)
GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error)
ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error)
InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (string, error)
Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error
Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error)
Expand Down
8 changes: 4 additions & 4 deletions pkg/storage/utils/eosfs/eosfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*pr
return grantList, nil
}

func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
u, err := getUser(ctx)
if err != nil {
return nil, err
Expand All @@ -517,7 +517,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.
// if path is home we need to add in the response any shadow folder in the shadow homedirectory.
if fs.conf.EnableHome {
if fs.isShareFolder(ctx, p) {
return fs.getMDShareFolder(ctx, p)
return fs.getMDShareFolder(ctx, p, mdKeys)
}
}

Expand All @@ -532,7 +532,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.
return fi, nil
}

func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.ResourceInfo, error) {
func (fs *eosfs) getMDShareFolder(ctx context.Context, p string, mdKeys []string) (*provider.ResourceInfo, error) {
u, err := getUser(ctx)
if err != nil {
return nil, err
Expand All @@ -551,7 +551,7 @@ func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.Reso
return fs.convertToFileReference(ctx, eosFileInfo), nil
}

func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) {
func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
log := appctx.GetLogger(ctx)
u, err := getUser(ctx)
if err != nil {
Expand Down
Loading

0 comments on commit 419de24

Please sign in to comment.