ocis driver: access shares, trash, revisions, metadata (cs3org#1155)

* initial version

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* introduce NodeInfo

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* fix metadata normalization

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* stick to github.com/google/uuid

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* simpler layout, parentid and name as ext attr

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* manually adding mimetypes is no longer needed

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* rename Wrap to NodeFrom*

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* refactor node implementation

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* fix delete and overwrite

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* initial sharing, store owner as ext attr, fix NewNode

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* separate files for metadada, revisions and recycle

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* list revisions

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* refactor and use ACE package

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* refactor node api

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* add license headers

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* add changelog

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* remove duplicate docs (moved to ace.go)

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* make linter happy

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* ocis: access shares

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* refactorings

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* fix upload

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* implement commit trash

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* implement RestoreRecycleItem

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* complete trash

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* address trash review

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* restore revision

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* download revision

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* fix litmus 409me

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* set and unset metadata

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* read metadata

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

changelog/unreleased/ocis-driver.md

@@ -0,0 +1,5 @@
+Enhancement: Introduce ocis storage driver
+
+We introduced a now storage driver `ocis` that deconstructs a filesystem and uses a node first approach to implement an efficient lookup of files by path as well as by file id.
+
+https://github.com/cs3org/reva/pull/559

internal/grpc/services/publicstorageprovider/publicstorageprovider.go

@@ -46,14 +46,13 @@ func init() {
 
 type config struct {
 	MountPath   string `mapstructure:"mount_path"`
-	MountID     string `mapstructure:"mount_id"`
 	GatewayAddr string `mapstructure:"gateway_addr"`
 }
 
 type service struct {
-	conf               *config
-	mountPath, mountID string
-	gateway            gateway.GatewayAPIClient
+	conf      *config
+	mountPath string
+	gateway   gateway.GatewayAPIClient
 }
 
 func (s *service) Close() error {
@@ -85,7 +84,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	}
 
 	mountPath := c.MountPath
-	mountID := c.MountID
 
 	gateway, err := pool.GetGatewayServiceClient(c.GatewayAddr)
 	if err != nil {
@@ -95,7 +93,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	service := &service{
 		conf:      c,
 		mountPath: mountPath,
-		mountID:   mountID,
 		gateway:   gateway,
 	}
 

pkg/storage/fs/loader/loader.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/cs3org/reva/pkg/storage/fs/eoshome"
 	_ "github.com/cs3org/reva/pkg/storage/fs/local"
 	_ "github.com/cs3org/reva/pkg/storage/fs/localhome"
+	_ "github.com/cs3org/reva/pkg/storage/fs/ocis"
 	_ "github.com/cs3org/reva/pkg/storage/fs/owncloud"
 	_ "github.com/cs3org/reva/pkg/storage/fs/s3"
 	// Add your own here

pkg/storage/fs/ocis/grants.go

@@ -0,0 +1,134 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package ocis
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/errtypes"
+	"github.com/cs3org/reva/pkg/storage/utils/ace"
+	"github.com/pkg/xattr"
+)
+
+func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
+	log := appctx.GetLogger(ctx)
+	log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()")
+	var node *Node
+	if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+
+	np := filepath.Join(fs.pw.Root, "nodes", node.ID)
+	e := ace.FromGrant(g)
+	principal, value := e.Marshal()
+	if err := xattr.Set(np, sharePrefix+principal, value); err != nil {
+		return err
+	}
+	return fs.tp.Propagate(ctx, node)
+}
+
+func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) {
+	var node *Node
+	if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+	log := appctx.GetLogger(ctx)
+	np := filepath.Join(fs.pw.Root, "nodes", node.ID)
+	var attrs []string
+	if attrs, err = xattr.List(np); err != nil {
+		log.Error().Err(err).Msg("error listing attributes")
+		return nil, err
+	}
+
+	log.Debug().Interface("attrs", attrs).Msg("read attributes")
+
+	aces := extractACEsFromAttrs(ctx, np, attrs)
+
+	grants = make([]*provider.Grant, 0, len(aces))
+	for i := range aces {
+		grants = append(grants, aces[i].Grant())
+	}
+
+	return grants, nil
+}
+
+func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
+	var node *Node
+	if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+
+	var attr string
+	if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
+		attr = sharePrefix + "g:" + g.Grantee.Id.OpaqueId
+	} else {
+		attr = sharePrefix + "u:" + g.Grantee.Id.OpaqueId
+	}
+
+	np := filepath.Join(fs.pw.Root, "nodes", node.ID)
+	if err = xattr.Remove(np, attr); err != nil {
+		return
+	}
+
+	return fs.tp.Propagate(ctx, node)
+}
+
+func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error {
+	return fs.AddGrant(ctx, ref, g)
+}
+
+// extractACEsFromAttrs reads ACEs in the list of attrs from the node
+func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) {
+	log := appctx.GetLogger(ctx)
+	entries = []*ace.ACE{}
+	for i := range attrs {
+		if strings.HasPrefix(attrs[i], sharePrefix) {
+			var value []byte
+			var err error
+			if value, err = xattr.Get(fsfn, attrs[i]); err != nil {
+				log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute")
+				continue
+			}
+			var e *ace.ACE
+			principal := attrs[i][len(sharePrefix):]
+			if e, err = ace.Unmarshal(principal, value); err != nil {
+				log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could unmarshal ace")
+				continue
+			}
+			entries = append(entries, e)
+		}
+	}
+	return
+}

pkg/storage/fs/ocis/metadata.go

@@ -0,0 +1,69 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package ocis
+
+import (
+	"context"
+	"path/filepath"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/errtypes"
+	"github.com/pkg/errors"
+	"github.com/pkg/xattr"
+)
+
+func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) {
+	n, err := fs.pw.NodeFromResource(ctx, ref)
+	if err != nil {
+		return errors.Wrap(err, "ocisfs: error resolving ref")
+	}
+
+	if !n.Exists {
+		err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
+		return err
+	}
+	nodePath := filepath.Join(fs.pw.Root, "nodes", n.ID)
+	for k, v := range md.Metadata {
+		attrName := metadataPrefix + k
+		if err = xattr.Set(nodePath, attrName, []byte(v)); err != nil {
+			return errors.Wrap(err, "ocisfs: could not set metadata attribute "+attrName+" to "+k)
+		}
+	}
+	return
+}
+
+func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) {
+	n, err := fs.pw.NodeFromResource(ctx, ref)
+	if err != nil {
+		return errors.Wrap(err, "ocisfs: error resolving ref")
+	}
+
+	if !n.Exists {
+		err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
+		return err
+	}
+	nodePath := filepath.Join(fs.pw.Root, "nodes", n.ID)
+	for i := range keys {
+		attrName := metadataPrefix + keys[i]
+		if err = xattr.Remove(nodePath, attrName); err != nil {
+			return errors.Wrap(err, "ocisfs: could not remove metadata attribute "+attrName)
+		}
+	}
+	return
+}