EOS spaces implementation (#2919)

* EOS spaces implementation

* Remove mount point prefix checks for absolute path refs

* Get storage space for full path when uploading

* Fix linting issues

* Remove hack for favorites path

* Handle quota requests for arbitrary resources in EOS

* Support userid claim in rest user driver

* Modularize setting storage provider ID

* Trim provider path for absolute path refs

* Use utils method to add path to opaque object

* TrimPrefix instead of TrimSuffix

Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de>

changelog/unreleased/eos-spaces.md

@@ -0,0 +1,3 @@
+Enhancement: EOS Spaces implementation
+
+https://github.com/cs3org/reva/pull/2919

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e
-	github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde
+	github.com/cs3org/go-cs3apis v0.0.0-20220512100524-551800f020d8
 	github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8
 	github.com/dgraph-io/ristretto v0.1.0
 	github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59

go.sum

@@ -218,6 +218,8 @@ github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJff
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0eBzpzVBIuczDhqVq50Nmjc7PGHHA9Y=
 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
+github.com/cs3org/go-cs3apis v0.0.0-20220512100524-551800f020d8 h1:31jPSD5BOjc4+h4xK1e+NFf34gtwHEexor3V7JbdcPM=
+github.com/cs3org/go-cs3apis v0.0.0-20220512100524-551800f020d8/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

internal/grpc/services/gateway/gateway.go

@@ -63,15 +63,16 @@ type config struct {
 	TokenManager                  string `mapstructure:"token_manager"`
 	// ShareFolder is the location where to create shares in the recipient's storage provider.
 	// FIXME get rid of ShareFolder, there are findByPath calls in the ocmshareporvider.go and usershareprovider.go
-	ShareFolder         string                            `mapstructure:"share_folder"`
-	DataTransfersFolder string                            `mapstructure:"data_transfers_folder"`
-	HomeMapping         string                            `mapstructure:"home_mapping"`
-	TokenManagers       map[string]map[string]interface{} `mapstructure:"token_managers"`
-	EtagCacheTTL        int                               `mapstructure:"etag_cache_ttl"`
-	AllowedUserAgents   map[string][]string               `mapstructure:"allowed_user_agents"` // map[path][]user-agent
-	CreateHomeCacheTTL  int                               `mapstructure:"create_home_cache_ttl"`
-	ProviderCacheTTL    int                               `mapstructure:"provider_cache_ttl"`
-	StatCacheTTL        int                               `mapstructure:"stat_cache_ttl"`
+	ShareFolder                  string                            `mapstructure:"share_folder"`
+	DataTransfersFolder          string                            `mapstructure:"data_transfers_folder"`
+	HomeMapping                  string                            `mapstructure:"home_mapping"`
+	TokenManagers                map[string]map[string]interface{} `mapstructure:"token_managers"`
+	EtagCacheTTL                 int                               `mapstructure:"etag_cache_ttl"`
+	AllowedUserAgents            map[string][]string               `mapstructure:"allowed_user_agents"` // map[path][]user-agent
+	CreateHomeCacheTTL           int                               `mapstructure:"create_home_cache_ttl"`
+	ProviderCacheTTL             int                               `mapstructure:"provider_cache_ttl"`
+	StatCacheTTL                 int                               `mapstructure:"stat_cache_ttl"`
+	UseCommonSpaceRootShareLogic bool                              `mapstructure:"use_common_space_root_share_logic"`
 	// MountCacheTTL       int                               `mapstructure:"mount_cache_ttl"`
 }
 

internal/grpc/services/gateway/usershareprovider.go

@@ -40,15 +40,15 @@ import (
 // TODO(labkode): add multi-phase commit logic when commit share or commit ref is enabled.
 func (s *svc) CreateShare(ctx context.Context, req *collaboration.CreateShareRequest) (*collaboration.CreateShareResponse, error) {
 	// Don't use the share manager when sharing a space root
-	if refIsSpaceRoot(req.ResourceInfo.Id) {
+	if !s.c.UseCommonSpaceRootShareLogic && refIsSpaceRoot(req.ResourceInfo.Id) {
 		return s.addSpaceShare(ctx, req)
 	}
 	return s.addShare(ctx, req)
 }
 
 func (s *svc) RemoveShare(ctx context.Context, req *collaboration.RemoveShareRequest) (*collaboration.RemoveShareResponse, error) {
 	key := req.Ref.GetKey()
-	if shareIsSpaceRoot(key) {
+	if !s.c.UseCommonSpaceRootShareLogic && shareIsSpaceRoot(key) {
 		return s.removeSpaceShare(ctx, key.ResourceId, key.Grantee)
 	}
 	return s.removeShare(ctx, req)

internal/grpc/services/storageprovider/storageprovider.go

@@ -498,8 +498,7 @@ func (s *service) CreateStorageSpace(ctx context.Context, req *provider.CreateSt
 	}
 
 	if resp.StorageSpace != nil {
-		resp.StorageSpace.Id.OpaqueId = storagespace.FormatStorageID(s.conf.MountID, resp.StorageSpace.Id.GetOpaqueId())
-		resp.StorageSpace.Root.StorageId = storagespace.FormatStorageID(s.conf.MountID, resp.StorageSpace.Root.GetStorageId())
+		s.addMissingStorageProviderID(resp.StorageSpace.Root, resp.StorageSpace.Id)
 	}
 	return resp, nil
 }
@@ -509,7 +508,8 @@ func (s *service) ListStorageSpaces(ctx context.Context, req *provider.ListStora
 		if f.Type == provider.ListStorageSpacesRequest_Filter_TYPE_ID {
 			_, id := storagespace.SplitStorageID(f.GetId().GetOpaqueId())
 			req.Filters[i].Term = &provider.ListStorageSpacesRequest_Filter_Id{Id: &provider.StorageSpaceId{OpaqueId: id}}
-			break
+		} else if f.Type == provider.ListStorageSpacesRequest_Filter_TYPE_PATH {
+			req.Filters[i].Term = &provider.ListStorageSpacesRequest_Filter_Path{Path: f.GetPath()}
 		}
 	}
 
@@ -551,8 +551,8 @@ func (s *service) ListStorageSpaces(ctx context.Context, req *provider.ListStora
 			log.Error().Str("service", "storageprovider").Str("driver", s.conf.Driver).Interface("space", sp).Msg("space is missing space id and root id")
 			continue
 		}
-		sp.Id.OpaqueId = storagespace.FormatStorageID(s.conf.MountID, sp.Id.GetOpaqueId())
-		sp.Root.StorageId = storagespace.FormatStorageID(s.conf.MountID, sp.Root.GetStorageId())
+
+		s.addMissingStorageProviderID(sp.Root, sp.Id)
 	}
 
 	return &provider.ListStorageSpacesResponse{
@@ -577,8 +577,7 @@ func (s *service) UpdateStorageSpace(ctx context.Context, req *provider.UpdateSt
 		return nil, err
 	}
 	if res.StorageSpace != nil {
-		res.StorageSpace.Id.OpaqueId = storagespace.FormatStorageID(s.conf.MountID, res.StorageSpace.Id.GetOpaqueId())
-		res.StorageSpace.Root.StorageId = storagespace.FormatStorageID(s.conf.MountID, res.StorageSpace.Root.GetStorageId())
+		s.addMissingStorageProviderID(res.StorageSpace.Root, res.StorageSpace.Id)
 	}
 	return res, nil
 }
@@ -687,7 +686,7 @@ func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*pro
 		Status: status.NewStatusFromErrType(ctx, "delete", err),
 		Opaque: &typesv1beta1.Opaque{
 			Map: map[string]*typesv1beta1.OpaqueEntry{
-				"opaque_id": &typesv1beta1.OpaqueEntry{Decoder: "plain", Value: []byte(md.Id.OpaqueId)},
+				"opaque_id": {Decoder: "plain", Value: []byte(md.Id.OpaqueId)},
 			},
 		},
 	}, nil
@@ -727,10 +726,15 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide
 		}, nil
 	}
 
-	if providerID == "" {
-		providerID = s.conf.MountID
+	// The storage driver might set the mount ID by itself, in which case skip this step
+	if mountID, _ := storagespace.SplitStorageID(md.Id.GetStorageId()); mountID == "" {
+		if providerID == "" {
+			providerID = s.conf.MountID
+		}
+
+		md.Id.StorageId = storagespace.FormatStorageID(providerID, md.Id.GetStorageId())
 	}
-	md.Id.StorageId = storagespace.FormatStorageID(providerID, md.Id.GetStorageId())
+
 	return &provider.StatResponse{
 		Status: status.NewOK(ctx),
 		Info:   md,
@@ -771,6 +775,7 @@ func (s *service) ListContainerStream(req *provider.ListContainerStreamRequest,
 	}
 
 	for _, md := range mds {
+		s.addMissingStorageProviderID(md.Id, nil)
 		res := &provider.ListContainerStreamResponse{
 			Info:   md,
 			Status: status.NewOK(ctx),
@@ -798,7 +803,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
 	}
 
 	for _, i := range res.Infos {
-		i.Id.StorageId = storagespace.FormatStorageID(s.conf.MountID, i.Id.GetStorageId())
+		s.addMissingStorageProviderID(i.Id, nil)
 	}
 	return res, nil
 }
@@ -867,6 +872,9 @@ func (s *service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p
 
 	// TODO(labkode): CRITICAL: fill recycle info with storage provider.
 	for _, item := range items {
+		if item.Ref != nil && item.Ref.ResourceId != nil {
+			s.addMissingStorageProviderID(item.Ref.GetResourceId(), nil)
+		}
 		res := &provider.ListRecycleStreamResponse{
 			RecycleItem: item,
 			Status:      status.NewOK(ctx),
@@ -909,7 +917,7 @@ func (s *service) ListRecycle(ctx context.Context, req *provider.ListRecycleRequ
 
 	for _, i := range items {
 		if i.Ref != nil && i.Ref.ResourceId != nil {
-			i.Ref.ResourceId.StorageId = storagespace.FormatStorageID(s.conf.MountID, i.Ref.GetResourceId().GetStorageId())
+			s.addMissingStorageProviderID(i.Ref.GetResourceId(), nil)
 		}
 	}
 	res := &provider.ListRecycleResponse{
@@ -1224,6 +1232,16 @@ func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (
 	return res, nil
 }
 
+func (s *service) addMissingStorageProviderID(resourceID *provider.ResourceId, spaceID *provider.StorageSpaceId) {
+	// The storage driver might set the mount ID by itself, in which case skip this step
+	if mountID, _ := storagespace.SplitStorageID(resourceID.GetStorageId()); mountID == "" {
+		resourceID.StorageId = storagespace.FormatStorageID(s.conf.MountID, resourceID.GetStorageId())
+		if spaceID != nil {
+			spaceID.OpaqueId = storagespace.FormatStorageID(s.conf.MountID, spaceID.GetOpaqueId())
+		}
+	}
+}
+
 func getFS(c *config) (storage.FS, error) {
 	if f, ok := registry.NewFuncs[c.Driver]; ok {
 		return f(c.Drivers[c.Driver])

internal/http/services/owncloud/ocdav/propfind/propfind.go

@@ -187,7 +187,6 @@ func (p *Handler) HandlePathPropfind(w http.ResponseWriter, r *http.Request, ns
 	fn := path.Join(ns, r.URL.Path) // TODO do we still need to jail if we query the registry about the spaces?
 
 	sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger()
-
 	pf, status, err := ReadPropfind(r.Body)
 	if err != nil {
 		sublog.Debug().Err(err).Msg("error reading propfind request")

internal/http/services/owncloud/ocdav/report.go

@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"io"
 	"net/http"
-	"path"
 
 	rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -108,10 +107,6 @@ func (s *svc) doFilterFiles(w http.ResponseWriter, r *http.Request, ff *reportFi
 				log.Error().Interface("stat_response", statRes).Msg("error getting resource info")
 				continue
 			}
-
-			// TODO: implement GetPath on storage provider to fix this
-			statRes.Info.Path = path.Join("/users/"+currentUser.Id.OpaqueId, statRes.Info.Path)
-
 			infos = append(infos, statRes.Info)
 		}
 

internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go

@@ -210,7 +210,6 @@ func (h *Handler) CreateShare(w http.ResponseWriter, r *http.Request) {
 		response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, errParsingSpaceReference.Error(), errParsingSpaceReference)
 		return
 	}
-
 	sublog := appctx.GetLogger(ctx).With().Interface("ref", ref).Logger()
 
 	statReq := provider.StatRequest{Ref: &ref}
@@ -275,7 +274,6 @@ func (h *Handler) CreateShare(w http.ResponseWriter, r *http.Request) {
 		}
 
 		h.mapUserIds(ctx, client, s)
-
 		if shareType == int(conversions.ShareTypeUser) {
 			res, err := client.GetUser(ctx, &userpb.GetUserRequest{
 				UserId: &userpb.UserId{
@@ -861,7 +859,7 @@ func (h *Handler) listSharesWithMe(w http.ResponseWriter, r *http.Request) {
 		data.State = mapState(rs.GetState())
 
 		if err := h.addFileInfo(ctx, data, info); err != nil {
-			log.Debug().Interface("received_share", rs).Interface("info", info).Interface("shareData", data).Err(err).Msg("could not add file info, skipping")
+			log.Debug().Interface("received_share", rs.Share.Id).Err(err).Msg("could not add file info, skipping")
 			continue
 		}
 		h.mapUserIds(r.Context(), client, data)

pkg/cbox/favorite/sql/sql.go

@@ -29,6 +29,7 @@ import (
 	ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
 	"github.com/cs3org/reva/v2/pkg/storage/favorite"
 	"github.com/cs3org/reva/v2/pkg/storage/favorite/registry"
+	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -82,6 +83,7 @@ func (m *mgr) ListFavorites(ctx context.Context, userID *user.UserId) ([]*provid
 		if err := rows.Scan(&info.StorageId, &info.OpaqueId); err != nil {
 			return nil, err
 		}
+		info.StorageId = storagespace.FormatStorageID(info.StorageId, info.OpaqueId)
 		infos = append(infos, &info)
 	}
 
@@ -93,18 +95,19 @@ func (m *mgr) ListFavorites(ctx context.Context, userID *user.UserId) ([]*provid
 
 func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error {
 	user := ctxpkg.ContextMustGetUser(ctx)
+	storageID, _ := storagespace.SplitStorageID(resourceInfo.Id.StorageId)
 
 	// The primary key is just the ID in the table, it should ideally be (uid, fileid_prefix, fileid, tag_key)
 	// For the time being, just check if the favorite already exists. If it does, return early
 	var id int
 	query := `SELECT id FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"`
-	if err := m.db.QueryRow(query, user.Id.OpaqueId, resourceInfo.Id.StorageId, resourceInfo.Id.OpaqueId).Scan(&id); err == nil {
+	if err := m.db.QueryRow(query, user.Id.OpaqueId, storageID, resourceInfo.Id.OpaqueId).Scan(&id); err == nil {
 		// Favorite is already set, return
 		return nil
 	}
 
 	query = `INSERT INTO cbox_metadata SET item_type=?, uid=?, fileid_prefix=?, fileid=?, tag_key="fav"`
-	vals := []interface{}{utils.ResourceTypeToItemInt(resourceInfo.Type), user.Id.OpaqueId, resourceInfo.Id.StorageId, resourceInfo.Id.OpaqueId}
+	vals := []interface{}{utils.ResourceTypeToItemInt(resourceInfo.Type), user.Id.OpaqueId, storageID, resourceInfo.Id.OpaqueId}
 	stmt, err := m.db.Prepare(query)
 	if err != nil {
 		return err
@@ -118,12 +121,14 @@ func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo
 
 func (m *mgr) UnsetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error {
 	user := ctxpkg.ContextMustGetUser(ctx)
+	storageID, _ := storagespace.SplitStorageID(resourceInfo.Id.StorageId)
+
 	stmt, err := m.db.Prepare(`DELETE FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"`)
 	if err != nil {
 		return err
 	}
 
-	res, err := stmt.Exec(user.Id.OpaqueId, resourceInfo.Id.StorageId, resourceInfo.Id.OpaqueId)
+	res, err := stmt.Exec(user.Id.OpaqueId, storageID, resourceInfo.Id.OpaqueId)
 	if err != nil {
 		return err
 	}

pkg/cbox/publicshare/sql/sql.go

@@ -39,6 +39,7 @@ import (
 	"github.com/cs3org/reva/v2/pkg/errtypes"
 	"github.com/cs3org/reva/v2/pkg/publicshare"
 	"github.com/cs3org/reva/v2/pkg/publicshare/manager/registry"
+	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"github.com/cs3org/reva/v2/pkg/utils"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
@@ -133,7 +134,7 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr
 	owner := conversions.FormatUserID(rInfo.Owner)
 	permissions := conversions.SharePermToInt(g.Permissions.Permissions)
 	itemType := conversions.ResourceTypeToItem(rInfo.Type)
-	prefix := rInfo.Id.StorageId
+	prefix, _ := storagespace.SplitStorageID(rInfo.Id.StorageId)
 	itemSource := rInfo.Id.OpaqueId
 	fileSource, err := strconv.ParseUint(itemSource, 10, 64)
 	if err != nil {
@@ -311,33 +312,44 @@ func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.Pu
 func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) {
 	uid := conversions.FormatUserID(u.Id)
 	query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner=? or uid_initiator=?) AND (share_type=?)"
-	var filterQuery string
+	var resourceFilters, ownerFilters, creatorFilters string
+	var resourceParams, ownerParams, creatorParams []interface{}
 	params := []interface{}{uid, uid, publicShareType}
 
-	for i, f := range filters {
+	for _, f := range filters {
 		switch f.Type {
 		case link.ListPublicSharesRequest_Filter_TYPE_RESOURCE_ID:
-			filterQuery += "(fileid_prefix=? AND item_source=?)"
-			if i != len(filters)-1 {
-				filterQuery += " AND "
+			if len(resourceFilters) != 0 {
+				resourceFilters += " OR "
 			}
-			params = append(params, f.GetResourceId().StorageId, f.GetResourceId().OpaqueId)
+			resourceFilters += "(fileid_prefix=? AND item_source=?)"
+			prefix, _ := storagespace.SplitStorageID(f.GetResourceId().StorageId)
+			resourceParams = append(resourceParams, prefix, f.GetResourceId().OpaqueId)
 		case link.ListPublicSharesRequest_Filter_TYPE_OWNER:
-			filterQuery += "(uid_owner=?)"
-			if i != len(filters)-1 {
-				filterQuery += " AND "
+			if len(ownerFilters) != 0 {
+				ownerFilters += " OR "
 			}
-			params = append(params, conversions.FormatUserID(f.GetOwner()))
+			ownerFilters += "(uid_owner=?)"
+			ownerParams = append(ownerParams, conversions.FormatUserID(f.GetOwner()))
 		case link.ListPublicSharesRequest_Filter_TYPE_CREATOR:
-			filterQuery += "(uid_initiator=?)"
-			if i != len(filters)-1 {
-				filterQuery += " AND "
+			if len(creatorFilters) != 0 {
+				creatorFilters += " OR "
 			}
-			params = append(params, conversions.FormatUserID(f.GetCreator()))
+			creatorFilters += "(uid_initiator=?)"
+			creatorParams = append(creatorParams, conversions.FormatUserID(f.GetCreator()))
 		}
 	}
-	if filterQuery != "" {
-		query = fmt.Sprintf("%s AND (%s)", query, filterQuery)
+	if resourceFilters != "" {
+		query = fmt.Sprintf("%s AND (%s)", query, resourceFilters)
+		params = append(params, resourceParams...)
+	}
+	if ownerFilters != "" {
+		query = fmt.Sprintf("%s AND (%s)", query, ownerFilters)
+		params = append(params, ownerParams...)
+	}
+	if creatorFilters != "" {
+		query = fmt.Sprintf("%s AND (%s)", query, creatorFilters)
+		params = append(params, creatorParams...)
 	}
 
 	rows, err := m.db.Query(query, params...)