From 87ac2e24d44f4b8394211761ae9b307e32def8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 2 Feb 2024 17:40:36 +0100 Subject: [PATCH] initial decomposedfs index cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- Makefile | 1 + go.mod | 2 +- go.sum | 4 +- ocis/pkg/command/decomposedfs.go | 123 ++++++++++++++++++ .../pkg/config/defaults/defaultconfig.go | 3 +- .../services/owncloud/ocdav/net/builders.go | 2 +- .../owncloud/ocdav/propfind/propfind.go | 1 + .../decomposedfs/spaceidindex/spaceidindex.go | 6 + vendor/modules.txt | 2 +- 9 files changed, 138 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 2e3550266b7..c731dd49b14 100644 --- a/Makefile +++ b/Makefile @@ -175,6 +175,7 @@ docs-generate: .PHONY: ci-go-generate ci-go-generate: + $(MAKE) --no-print-directory -C protobuf ci-go-generate || exit 1 @for mod in $(OCIS_MODULES); do \ $(MAKE) --no-print-directory -C $$mod ci-go-generate || exit 1; \ done diff --git a/go.mod b/go.mod index 7ba7837ae00..f08055e8faf 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.10 github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52 + github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e diff --git a/go.sum b/go.sum index 80a34ec6ab9..3bfbcc3b9bc 100644 --- a/go.sum +++ b/go.sum @@ -1019,8 +1019,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52 h1:Jeh8q6WKl4gcK7GMayn56y1uxbw91XvyuZcvY7SiDRk= -github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= +github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c h1:LAU3JemlUr6Ti6hMaEzSnOGmaZKRzqLuu2CqkbZF9Yw= +github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/ocis/pkg/command/decomposedfs.go b/ocis/pkg/command/decomposedfs.go index 7ee651e3e57..2ebc01a5fd0 100644 --- a/ocis/pkg/command/decomposedfs.go +++ b/ocis/pkg/command/decomposedfs.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "path/filepath" "sort" "strings" @@ -16,8 +17,10 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/store" @@ -35,6 +38,7 @@ func DecomposedfsCommand(cfg *config.Config) *cli.Command { Subcommands: []*cli.Command{ metadataCmd(cfg), checkCmd(cfg), + indexCmd(cfg), }, } } @@ -378,3 +382,122 @@ func attribToString(attrib []byte) string { } return `"` + string(attrib) + `"` } + +func indexCmd(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "index", + Usage: `cli tool to check the space indexes`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "root", + Aliases: []string{"r"}, + Required: true, + Usage: "Path to the root directory of the decomposedfs", + }, + // TODO String flag for the type? + // list values by default? + &cli.BoolFlag{ + Name: "repair", + Usage: "add all spaces to the indexes", + }, + }, + Action: index, + } +} + +func index(c *cli.Context) error { + rootFlag := c.String("root") + //repairFlag := c.Bool("repair") + + files, err := filepath.Glob(filepath.Join(rootFlag, "spaces", "*", "*")) + if err != nil { + return err + } + + lu, _ := getBackend(c) + + byuserid := map[string]map[string]string{} + bygroupid := map[string]map[string]string{} + bytype := map[string]map[string]string{} + + for _, file := range files { + spaceID := filepath.Base(filepath.Dir(file)) + filepath.Base(file) + fmt.Println("space " + spaceID) + n, err := node.ReadNode(c.Context, lu, spaceID, spaceID, true, nil, true) + if err != nil { + fmt.Println(" could not read node: " + err.Error()) + continue + } + spacetype, err := n.SpaceRoot.XattrString(c.Context, prefixes.SpaceTypeAttr) + if err != nil { + fmt.Println(" could not read space type: " + err.Error()) + continue + } + fmt.Println(" type " + spacetype) + target := "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) + if bytype[spacetype] == nil { + bytype[spacetype] = make(map[string]string) + } + bytype[spacetype][spaceID] = target + grants, err := n.ListGrants(c.Context) + if err != nil { + fmt.Println(" could not list grants: " + err.Error()) + continue + } + for _, grant := range grants { + if grant.GetGrantee().GetUserId() != nil { + userid := grant.GetGrantee().GetUserId().GetOpaqueId() + fmt.Println(" user grant: " + grant.GetGrantee().GetUserId().GetOpaqueId()) + if byuserid[userid] == nil { + byuserid[userid] = make(map[string]string) + } + byuserid[userid][spaceID] = target + } + if grant.GetGrantee().GetGroupId() != nil { + groupid := grant.GetGrantee().GetGroupId().GetOpaqueId() + fmt.Println(" group grant: " + groupid) + if bygroupid[groupid] == nil { + bygroupid[groupid] = make(map[string]string) + } + bygroupid[groupid][spaceID] = target + } + } + } + // TODO use a flag and lock to completely lock the indexes and rewrite them + + fmt.Println("rebuilding by-user-id indexes") + userSpaceIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-user-id") + err = userSpaceIndex.Init() + if err != nil { + return err + } + for userid, spaces := range byuserid { + fmt.Print(" ", userid, " ", spaces, "\n") + userSpaceIndex.AddAll(userid, spaces) + } + + fmt.Println("rebuilding by-group-id indexes") + groupSpaceIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-group-id") + err = groupSpaceIndex.Init() + if err != nil { + return err + } + for groupid, spaces := range bygroupid { + fmt.Print(" ", groupid, " ", spaces, "\n") + groupSpaceIndex.AddAll(groupid, spaces) + } + + fmt.Println("rebuilding by-type indexes") + spaceTypeIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-type") + err = spaceTypeIndex.Init() + if err != nil { + return err + } + for spaceType, spaces := range bytype { + fmt.Print(" ", spaceType, " ", spaces, "\n") + spaceTypeIndex.AddAll(spaceType, spaces) + } + // FIXME type might be share which requires iterating over all files or manually adding a space after dumping a list of shares from the share manager + + return nil +} diff --git a/services/postprocessing/pkg/config/defaults/defaultconfig.go b/services/postprocessing/pkg/config/defaults/defaultconfig.go index 11cf3f02af0..d29434f84ea 100644 --- a/services/postprocessing/pkg/config/defaults/defaultconfig.go +++ b/services/postprocessing/pkg/config/defaults/defaultconfig.go @@ -35,9 +35,10 @@ func DefaultConfig() *config.Config { MaxRetries: 14, }, Store: config.Store{ - Store: "memory", + Store: "nats-js-kv", Database: "postprocessing", Table: "postprocessing", + Nodes: []string{"127.0.0.1:9233"}, }, } } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net/builders.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net/builders.go index 0498072b6c3..be7b8fdeecf 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net/builders.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net/builders.go @@ -27,7 +27,7 @@ import ( // ContentDispositionAttachment builds a ContentDisposition Attachment header with various filename encodings func ContentDispositionAttachment(filename string) string { - return "attachment; filename*=UTF-8''" + filename + "; filename=\"" + filename + "\"" + return "attachment; filename*=UTF-8''\"" + filename + "\"; filename=\"" + filename + "\"" } // RFC1123Z formats a CS3 Timestamp to be used in HTTP headers like Last-Modified diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go index fc5764738e9..027f8c73b16 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go @@ -579,6 +579,7 @@ func (p *Handler) getResourceInfos(ctx context.Context, w http.ResponseWriter, r // adjust path info.Path = filepath.Join(spacePath, spaceRef.Path) + info.Name = filepath.Base(info.Path) spaceMap[info] = spaceData{Ref: spaceRef, SpaceType: space.SpaceType} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go index 4d9add38951..d9dc9c35973 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go @@ -50,10 +50,16 @@ func (i *Index) Load(index string) (map[string]string, error) { } // Add adds an entry to an index +// Consider calling AddAll() when trying to add multiple entries as every Add call has to lock the index func (i *Index) Add(index, key string, value string) error { return i.updateIndex(index, map[string]string{key: value}, []string{}) } +// AddAll adds multiple entries to the index +func (i *Index) AddAll(index string, m map[string]string) error { + return i.updateIndex(index, m, []string{}) +} + // Remove removes an entry from the index func (i *Index) Remove(index, key string) error { return i.updateIndex(index, map[string]string{}, []string{key}) diff --git a/vendor/modules.txt b/vendor/modules.txt index 7850ae139b8..316219fdd7d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52 +# github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime