initial decomposedfs index cmd

Makefile

@@ -175,6 +175,7 @@ docs-generate:
 
 .PHONY: ci-go-generate
 ci-go-generate:
+    $(MAKE) --no-print-directory -C protobuf ci-go-generate || exit 1
 	@for mod in $(OCIS_MODULES); do \
         $(MAKE) --no-print-directory -C $$mod ci-go-generate || exit 1; \
     done

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.10
 	github.com/coreos/go-oidc/v3 v3.9.0
 	github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781
-	github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52
+	github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c
 	github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25
 	github.com/disintegration/imaging v1.6.2
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e

go.sum

@@ -1019,8 +1019,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c=
 github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME=
 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY=
 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
-github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52 h1:Jeh8q6WKl4gcK7GMayn56y1uxbw91XvyuZcvY7SiDRk=
-github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4=
+github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c h1:LAU3JemlUr6Ti6hMaEzSnOGmaZKRzqLuu2CqkbZF9Yw=
+github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=

ocis/pkg/command/decomposedfs.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -16,8 +17,10 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree"
 	"github.com/cs3org/reva/v2/pkg/storagespace"
 	"github.com/cs3org/reva/v2/pkg/store"
@@ -35,6 +38,7 @@ func DecomposedfsCommand(cfg *config.Config) *cli.Command {
 		Subcommands: []*cli.Command{
 			metadataCmd(cfg),
 			checkCmd(cfg),
+			indexCmd(cfg),
 		},
 	}
 }
@@ -378,3 +382,122 @@ func attribToString(attrib []byte) string {
 	}
 	return `"` + string(attrib) + `"`
 }
+
+func indexCmd(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "index",
+		Usage: `cli tool to check the space indexes`,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "root",
+				Aliases:  []string{"r"},
+				Required: true,
+				Usage:    "Path to the root directory of the decomposedfs",
+			},
+			// TODO String flag for the type?
+			// list values by default?
+			&cli.BoolFlag{
+				Name:  "repair",
+				Usage: "add all spaces to the indexes",
+			},
+		},
+		Action: index,
+	}
+}
+
+func index(c *cli.Context) error {
+	rootFlag := c.String("root")
+	//repairFlag := c.Bool("repair")
+
+	files, err := filepath.Glob(filepath.Join(rootFlag, "spaces", "*", "*"))
+	if err != nil {
+		return err
+	}
+
+	lu, _ := getBackend(c)
+
+	byuserid := map[string]map[string]string{}
+	bygroupid := map[string]map[string]string{}
+	bytype := map[string]map[string]string{}
+
+	for _, file := range files {
+		spaceID := filepath.Base(filepath.Dir(file)) + filepath.Base(file)
+		fmt.Println("space " + spaceID)
+		n, err := node.ReadNode(c.Context, lu, spaceID, spaceID, true, nil, true)
+		if err != nil {
+			fmt.Println("  could not read node: " + err.Error())
+			continue
+		}
+		spacetype, err := n.SpaceRoot.XattrString(c.Context, prefixes.SpaceTypeAttr)
+		if err != nil {
+			fmt.Println("  could not read space type: " + err.Error())
+			continue
+		}
+		fmt.Println("  type " + spacetype)
+		target := "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2)
+		if bytype[spacetype] == nil {
+			bytype[spacetype] = make(map[string]string)
+		}
+		bytype[spacetype][spaceID] = target
+		grants, err := n.ListGrants(c.Context)
+		if err != nil {
+			fmt.Println("  could not list grants: " + err.Error())
+			continue
+		}
+		for _, grant := range grants {
+			if grant.GetGrantee().GetUserId() != nil {
+				userid := grant.GetGrantee().GetUserId().GetOpaqueId()
+				fmt.Println("  user grant: " + grant.GetGrantee().GetUserId().GetOpaqueId())
+				if byuserid[userid] == nil {
+					byuserid[userid] = make(map[string]string)
+				}
+				byuserid[userid][spaceID] = target
+			}
+			if grant.GetGrantee().GetGroupId() != nil {
+				groupid := grant.GetGrantee().GetGroupId().GetOpaqueId()
+				fmt.Println("  group grant: " + groupid)
+				if bygroupid[groupid] == nil {
+					bygroupid[groupid] = make(map[string]string)
+				}
+				bygroupid[groupid][spaceID] = target
+			}
+		}
+	}
+	// TODO use a flag and lock to completely lock the indexes and rewrite them
+
+	fmt.Println("rebuilding by-user-id indexes")
+	userSpaceIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-user-id")
+	err = userSpaceIndex.Init()
+	if err != nil {
+		return err
+	}
+	for userid, spaces := range byuserid {
+		fmt.Print("  ", userid, " ", spaces, "\n")
+		userSpaceIndex.AddAll(userid, spaces)
+	}
+
+	fmt.Println("rebuilding by-group-id indexes")
+	groupSpaceIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-group-id")
+	err = groupSpaceIndex.Init()
+	if err != nil {
+		return err
+	}
+	for groupid, spaces := range bygroupid {
+		fmt.Print("  ", groupid, " ", spaces, "\n")
+		groupSpaceIndex.AddAll(groupid, spaces)
+	}
+
+	fmt.Println("rebuilding by-type indexes")
+	spaceTypeIndex := spaceidindex.New(filepath.Join(rootFlag, "indexes"), "by-type")
+	err = spaceTypeIndex.Init()
+	if err != nil {
+		return err
+	}
+	for spaceType, spaces := range bytype {
+		fmt.Print("  ", spaceType, " ", spaces, "\n")
+		spaceTypeIndex.AddAll(spaceType, spaces)
+	}
+	// FIXME type might be share which requires iterating over all files or manually adding a space after dumping a list of shares from the share manager
+
+	return nil
+}

services/postprocessing/pkg/config/defaults/defaultconfig.go

@@ -35,9 +35,10 @@ func DefaultConfig() *config.Config {
 			MaxRetries:           14,
 		},
 		Store: config.Store{
-			Store:    "memory",
+			Store:    "nats-js-kv",
 			Database: "postprocessing",
 			Table:    "postprocessing",
+			Nodes:    []string{"127.0.0.1:9233"},
 		},
 	}
 }

vendor/modules.txt

@@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
 github.com/cs3org/go-cs3apis/cs3/types/v1beta1
-# github.com/cs3org/reva/v2 v2.18.1-0.20240205065033-2c21ada2ae52
+# github.com/cs3org/reva/v2 v2.18.1-0.20240206134338-1a380e00b12c
 ## explicit; go 1.21
 github.com/cs3org/reva/v2/cmd/revad/internal/grace
 github.com/cs3org/reva/v2/cmd/revad/runtime