add a storage template package to define the user home directory (#510)

cmd/revad/runtime/loader.go

@@ -32,7 +32,6 @@ import (
 	_ "github.com/cs3org/reva/pkg/publicshare/manager/loader"
 	_ "github.com/cs3org/reva/pkg/share/manager/loader"
 	_ "github.com/cs3org/reva/pkg/storage/fs/loader"
-	_ "github.com/cs3org/reva/pkg/storage/pw/loader"
 	_ "github.com/cs3org/reva/pkg/storage/registry/loader"
 	_ "github.com/cs3org/reva/pkg/token/manager/loader"
 	_ "github.com/cs3org/reva/pkg/user/manager/loader"

go.mod

@@ -4,6 +4,9 @@ require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	github.com/BurntSushi/toml v0.3.1
+	github.com/Masterminds/goutils v1.1.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/aws/aws-sdk-go v1.29.9
 	github.com/cheggaaa/pb v1.0.28
 	github.com/coreos/go-oidc v2.2.1+incompatible
@@ -16,10 +19,13 @@ require (
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/uuid v1.1.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
+	github.com/huandu/xstrings v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.4 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/ory/fosite v0.30.2
 	github.com/pkg/errors v0.9.1

go.sum

@@ -6,6 +6,12 @@ contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE
 contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
+github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
@@ -14,8 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzs
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.29.5 h1:PddgnlgWgNI6x/weTnfk1fGYkhcs363gieDzK+Cf91Q=
-github.com/aws/aws-sdk-go v1.29.5/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
 github.com/aws/aws-sdk-go v1.29.9 h1:PHq9ddjfZYfCOXyqHKiCZ1CHRAk7nXhV7WTqj5l+bmQ=
 github.com/aws/aws-sdk-go v1.29.9/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@@ -79,6 +83,10 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo=
+github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo=
 github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
@@ -96,8 +104,12 @@ github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=

internal/grpc/services/storageprovider/storageprovider.go

@@ -34,7 +34,6 @@ import (
 	"github.com/cs3org/reva/pkg/rgrpc/status"
 	"github.com/cs3org/reva/pkg/storage"
 	"github.com/cs3org/reva/pkg/storage/fs/registry"
-	pwregistry "github.com/cs3org/reva/pkg/storage/pw/registry"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
@@ -134,10 +133,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	if err != nil {
 		return nil, err
 	}
-	pw, err := getPW(c)
-	if err != nil {
-		return nil, err
-	}
 
 	// parse data server url
 	u, err := url.Parse(c.DataServerURL)
@@ -158,7 +153,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	service := &service{
 		conf:          c,
 		storage:       fs,
-		pathWrapper:   pw,
 		tmpFolder:     tmpFolder,
 		mountPath:     mountPath,
 		mountID:       mountID,
@@ -789,16 +783,6 @@ func getFS(c *config) (storage.FS, error) {
 	return nil, fmt.Errorf("driver not found: %s", c.Driver)
 }
 
-func getPW(c *config) (storage.PathWrapper, error) {
-	if c.PathWrapper == "" {
-		return nil, nil
-	}
-	if f, ok := pwregistry.NewFuncs[c.PathWrapper]; ok {
-		return f(c.PathWrappers[c.PathWrapper])
-	}
-	return nil, fmt.Errorf("path wrapper not found: %s", c.Driver)
-}
-
 func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) {
 	if ref.GetId() != nil {
 		idRef := &provider.Reference{

pkg/storage/fs/eos/eos.go

@@ -31,14 +31,13 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cs3org/reva/pkg/storage/fs/registry"
-	"github.com/cs3org/reva/pkg/storage/helper"
-
 	"github.com/cs3org/reva/pkg/appctx"
 	"github.com/cs3org/reva/pkg/eosclient"
 	"github.com/cs3org/reva/pkg/mime"
 	"github.com/cs3org/reva/pkg/storage"
 	"github.com/cs3org/reva/pkg/storage/acl"
+	"github.com/cs3org/reva/pkg/storage/fs/registry"
+	"github.com/cs3org/reva/pkg/storage/templates"
 	"github.com/cs3org/reva/pkg/user"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
@@ -56,10 +55,8 @@ func init() {
 var hiddenReg = regexp.MustCompile(`\.sys\..#.`)
 
 type eosStorage struct {
-	c             *eosclient.Client
-	mountpoint    string
-	showHiddenSys bool
-	conf          *config
+	c    *eosclient.Client
+	conf *config
 }
 
 func parseConfig(m map[string]interface{}) (*config, error) {
@@ -95,6 +92,21 @@ type config struct {
 	// Defaults to os.TempDir()
 	CacheDirectory string `mapstructure:"cache_directory"`
 
+	// SecProtocol specifies the xrootd security protocol to use between the server and EOS.
+	SecProtocol string `mapstructure:"sec_protocol"`
+
+	// Keytab specifies the location of the keytab to use to authenticate to EOS.
+	Keytab string `mapstructure:"keytab"`
+
+	// SingleUsername is the username to use when SingleUserMode is enabled
+	SingleUsername string `mapstructure:"single_username"`
+
+	// UserLayout wraps the internal path with user information.
+	// Example: if conf.Namespace is /eos/user and received path is /docs
+	// and the UserLayout is {{.Username}} the internal path will be:
+	// /eos/user/<username>/docs
+	UserLayout string `mapstructure:"user_layout"`
+
 	// Enables logging of the commands executed
 	// Defaults to false
 	EnableLogging bool `mapstructure:"enable_logging"`
@@ -111,18 +123,6 @@ type config struct {
 
 	// EnableHome enables the creation of home directories.
 	EnableHome bool `mapstructure:"enable_home"`
-
-	// SecProtocol specifies the xrootd security protocol to use between the server and EOS.
-	SecProtocol string `mapstructure:"sec_protocol"`
-
-	// Keytab specifies the location of the keytab to use to authenticate to EOS.
-	Keytab string `mapstructure:"keytab"`
-
-	// SingleUsername is the username to use when SingleUserMode is enabled
-	SingleUsername string `mapstructure:"single_username"`
-
-	// Layout
-	Layout string `mapstructure:"layout"`
 }
 
 func getUser(ctx context.Context) (*userpb.User, error) {
@@ -159,10 +159,6 @@ func (c *config) init() {
 	if c.CacheDirectory == "" {
 		c.CacheDirectory = os.TempDir()
 	}
-
-	if c.Layout == "" {
-		c.Layout = "{{.Username}}"
-	}
 }
 
 // New returns a new implementation of the storage.FS interface that connects to EOS.
@@ -196,42 +192,50 @@ func New(m map[string]interface{}) (storage.FS, error) {
 	eosClient := eosclient.New(eosClientOpts)
 
 	eosStorage := &eosStorage{
-		c:             eosClient,
-		mountpoint:    c.Namespace,
-		showHiddenSys: c.ShowHiddenSysFiles,
-		conf:          c,
+		c:    eosClient,
+		conf: c,
 	}
 
 	return eosStorage, nil
 }
 
-func (fs *eosStorage) getHomeForUser(u *userpb.User) (string, error) {
-	userhome, err := helper.GetUserHomePath(u, fs.conf.Layout)
-	if err != nil {
-		return "", err
-	}
-
-	home := path.Join(fs.mountpoint, userhome)
-	return home, nil
-}
-
 func (fs *eosStorage) Shutdown(ctx context.Context) error {
 	// TODO(labkode): in a grpc implementation we can close connections.
 	return nil
 }
 
-func (fs *eosStorage) getInternalPath(ctx context.Context, fn string) string {
-	internalPath := path.Join(fs.mountpoint, fn)
-	return internalPath
+func (fs *eosStorage) wrap(ctx context.Context, fn string) (internal string) {
+	if fs.conf.EnableHome && fs.conf.UserLayout != "" {
+		u, err := getUser(ctx)
+		if err != nil {
+			err = errors.Wrap(err, "eos: wrap: no user in ctx and home is enabled")
+			panic(err)
+		}
+		layout := templates.WithUser(u, fs.conf.UserLayout)
+		internal = path.Join(fs.conf.Namespace, layout, fn)
+	} else {
+		internal = path.Join(fs.conf.Namespace, fn)
+	}
+	return
 }
 
-func (fs *eosStorage) removeNamespace(ctx context.Context, np string) string {
-	p := strings.TrimPrefix(np, fs.mountpoint)
-	if p == "" {
-		p = "/"
+func (fs *eosStorage) unwrap(ctx context.Context, np string) (external string) {
+	if fs.conf.EnableHome && fs.conf.UserLayout != "" {
+		u, err := getUser(ctx)
+		if err != nil {
+			err = errors.Wrap(err, "eos: unwrap: no user in ctx and home is enabled")
+			panic(err)
+		}
+		layout := templates.WithUser(u, fs.conf.UserLayout)
+		trim := path.Join(fs.conf.Namespace, layout)
+		external = strings.TrimPrefix(np, trim)
+	} else {
+		external = strings.TrimPrefix(np, fs.conf.Namespace)
+		if external == "" {
+			external = "/"
+		}
 	}
-
-	return p
+	return
 }
 
 func (fs *eosStorage) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) {
@@ -259,7 +263,7 @@ func (fs *eosStorage) GetPathByID(ctx context.Context, id *provider.ResourceId)
 // resolve takes in a request path or request id and converts it to a internal path.
 func (fs *eosStorage) resolve(ctx context.Context, u *userpb.User, ref *provider.Reference) (string, error) {
 	if ref.GetPath() != "" {
-		return fs.getInternalPath(ctx, ref.GetPath()), nil
+		return fs.wrap(ctx, ref.GetPath()), nil
 	}
 
 	if ref.GetId() != nil {
@@ -556,7 +560,7 @@ func (fs *eosStorage) ListFolder(ctx context.Context, ref *provider.Reference) (
 	finfos := []*provider.ResourceInfo{}
 	for _, eosFileInfo := range eosFileInfos {
 		// filter out sys files
-		if !fs.showHiddenSys {
+		if !fs.conf.ShowHiddenSysFiles {
 			base := path.Base(eosFileInfo.File)
 			if hiddenReg.MatchString(base) {
 				continue
@@ -577,26 +581,25 @@ func (fs *eosStorage) GetQuota(ctx context.Context) (int, int, error) {
 }
 
 func (fs *eosStorage) GetHome(ctx context.Context) (string, error) {
-	u, err := getUser(ctx)
-	if err != nil {
-		return "", errors.Wrap(err, "eos: no user in ctx")
-	}
-
-	home, err := fs.getHomeForUser(u)
-	if err != nil {
-		return "", err
+	if !fs.conf.EnableHome {
+		return "", errtypes.NotSupported("eos: get home not supported")
 	}
 
+	home := fs.wrap(ctx, "/")
 	return home, nil
 }
 
 func (fs *eosStorage) CreateHome(ctx context.Context) error {
+	if !fs.conf.EnableHome {
+		return errtypes.NotSupported("eos: create home not supported")
+	}
+
 	u, err := getUser(ctx)
 	if err != nil {
 		return errors.Wrap(err, "eos: no user in ctx")
 	}
 
-	home, err := fs.getHomeForUser(u)
+	home := fs.wrap(ctx, "/")
 	if err != nil {
 		return err
 	}
@@ -665,7 +668,7 @@ func (fs *eosStorage) CreateDir(ctx context.Context, fn string) error {
 		return errors.Wrap(err, "eos: no user in ctx")
 	}
 
-	fn = fs.getInternalPath(ctx, fn)
+	fn = fs.wrap(ctx, fn)
 	return fs.c.CreateDir(ctx, u.Username, fn)
 }
 
@@ -810,7 +813,7 @@ func (fs *eosStorage) DownloadRevision(ctx context.Context, ref *provider.Refere
 		return nil, errors.Wrap(err, "eos: error resolving reference")
 	}
 
-	fn = fs.getInternalPath(ctx, fn)
+	fn = fs.wrap(ctx, fn)
 	return fs.c.ReadVersion(ctx, u.Username, fn, revisionKey)
 }
 
@@ -855,7 +858,7 @@ func (fs *eosStorage) ListRecycle(ctx context.Context) ([]*provider.RecycleItem,
 	}
 	recycleEntries := []*provider.RecycleItem{}
 	for _, entry := range eosDeletedEntries {
-		if !fs.showHiddenSys {
+		if !fs.conf.ShowHiddenSysFiles {
 			base := path.Base(entry.RestorePath)
 			if hiddenReg.MatchString(base) {
 				continue
@@ -878,7 +881,7 @@ func (fs *eosStorage) RestoreRecycleItem(ctx context.Context, key string) error
 
 func (fs *eosStorage) convertToRecycleItem(ctx context.Context, eosDeletedItem *eosclient.DeletedEntry) *provider.RecycleItem {
 	recycleItem := &provider.RecycleItem{
-		Path:         fs.removeNamespace(ctx, eosDeletedItem.RestorePath),
+		Path:         fs.unwrap(ctx, eosDeletedItem.RestorePath),
 		Key:          eosDeletedItem.RestoreKey,
 		Size:         eosDeletedItem.Size,
 		DeletionTime: &types.Timestamp{Seconds: eosDeletedItem.DeletionMTime / 1000}, // TODO(labkode): check if eos time is millis or nanos
@@ -903,7 +906,7 @@ func (fs *eosStorage) convertToRevision(ctx context.Context, eosFileInfo *eoscli
 }
 
 func (fs *eosStorage) convertToResourceInfo(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo {
-	path := fs.removeNamespace(ctx, eosFileInfo.File)
+	path := fs.unwrap(ctx, eosFileInfo.File)
 	size := eosFileInfo.Size
 	if eosFileInfo.IsDir {
 		size = eosFileInfo.TreeSize