Merge pull request #3602 from owncloud/metadata-gateway

introduce metadata gateway

changelog/unreleased/metadata-gateway.md

@@ -0,0 +1,5 @@
+Enhancement: wrap metadata storage with dedicated reva gateway
+
+We wrapped the metadata storage in a minimal reva instance with a dedicated gateway, including static storage registry, static auth registry, in memory userprovider, machine authprovider and demo permissions service. This allows us to preconfigure the service user for the ocis settings service, share and public share providers.
+
+https://github.com/owncloud/ocis/pull/3602

changelog/unreleased/update-reva.md

@@ -7,6 +7,7 @@ Updated reva to version 2.x.x. This update includes:
 https://github.com/owncloud/ocis/pull/3552
 https://github.com/owncloud/ocis/pull/3570
 https://github.com/owncloud/ocis/pull/3601
+https://github.com/owncloud/ocis/pull/3602
 https://github.com/owncloud/ocis/pull/3605
 https://github.com/owncloud/ocis/pull/3611
 https://github.com/owncloud/ocis/issues/3621

extensions/idm/ldif/base.ldif.tmpl

@@ -40,17 +40,4 @@ userPassword:: {{ .Password }}
 
 {{ end -}}
 
-## Service user for the settings service
-dn: uid=95cb8724-03b2-11eb-a0a6-c33ef8ef53ad,ou=users,o=libregraph-idm
-objectClass: inetOrgPerson
-objectClass: organizationalPerson
-objectClass: ownCloud
-objectClass: person
-objectClass: top
-uid: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
-givenName: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
-sn: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
-cn: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
-displayName: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
-ownCloudUUID: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad
 

extensions/settings/pkg/config/config.go

@@ -40,6 +40,6 @@ type Metadata struct {
 	StorageAddress string `yaml:"storage_addr" env:"STORAGE_GRPC_ADDR"`
 
 	ServiceUserID     string `yaml:"service_user_id" env:"METADATA_SERVICE_USER_UUID"`
-	ServiceUserIDP    string `yaml:"service_user_idp" env:"OCIS_URL;METADATA_SERVICE_USER_IDP"`
+	ServiceUserIDP    string `yaml:"service_user_idp" env:"METADATA_SERVICE_USER_IDP"`
 	MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY"`
 }

extensions/settings/pkg/config/defaults/defaultconfig.go

@@ -50,10 +50,9 @@ func DefaultConfig() *config.Config {
 		},
 
 		Metadata: config.Metadata{
-			GatewayAddress: "127.0.0.1:9142",
+			GatewayAddress: "127.0.0.1:9215", // metadata storage
 			StorageAddress: "127.0.0.1:9215",
-			ServiceUserID:  "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			ServiceUserIDP: "https://localhost:9200",
+			ServiceUserIDP: "internal",
 		},
 	}
 }
@@ -93,6 +92,10 @@ func EnsureDefaults(cfg *config.Config) {
 	if cfg.Metadata.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" {
 		cfg.Metadata.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey
 	}
+
+	if cfg.Metadata.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" {
+		cfg.Metadata.ServiceUserID = cfg.Commons.MetadataUserID
+	}
 }
 
 func Sanitize(cfg *config.Config) {

extensions/settings/pkg/service/v0/settings.go

@@ -6,9 +6,6 @@ import (
 )
 
 const (
-	// BundleUUIDRoleMetadata represents the metadata user role
-	BundleUUIDRoleMetadata = "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad"
-
 	// BundleUUIDRoleAdmin represents the admin role
 	BundleUUIDRoleAdmin = "71881883-1768-46bd-a24d-a356a2afdf7f"
 
@@ -532,34 +529,11 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest {
 				},
 			},
 		},
-		{
-			BundleId: BundleUUIDRoleMetadata,
-			Setting: &settingsmsg.Setting{
-				Id:          CreateSpacePermissionID,
-				Name:        CreateSpacePermissionName,
-				DisplayName: "Create own Space",
-				Description: "This permission allows to create a space owned by the current user.",
-				Resource: &settingsmsg.Resource{
-					Type: settingsmsg.Resource_TYPE_SYSTEM, // TODO resource type space? self? me? own?
-				},
-				Value: &settingsmsg.Setting_PermissionValue{
-					PermissionValue: &settingsmsg.Permission{
-						Operation:  settingsmsg.Permission_OPERATION_CREATE,
-						Constraint: settingsmsg.Permission_CONSTRAINT_OWN,
-					},
-				},
-			},
-		},
 	}
 }
 
 func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment {
 	return []*settingsmsg.UserRoleAssignment{
-		// accounts service user for the metadata user is allowed to create spaces
-		{
-			AccountUuid: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			RoleId:      BundleUUIDRoleAdmin,
-		},
 		// default admin users
 		{
 			AccountUuid: "058bff95-6708-4fe5-91e4-9ea3d377588b",

extensions/settings/pkg/store/defaults/defaults.go

@@ -17,9 +17,6 @@ const (
 	// BundleUUIDRoleGuest represents the guest role.
 	BundleUUIDRoleGuest = "38071a68-456a-4553-846a-fa67bf5596cc"
 
-	// BundleUUIDRoleMetadata represents the metadata user role
-	BundleUUIDRoleMetadata = "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad"
-
 	// RoleManagementPermissionID is the hardcoded setting UUID for the role management permission
 	RoleManagementPermissionID string = "a53e601e-571f-4f86-8fec-d4576ef49c62"
 	// RoleManagementPermissionName is the hardcoded setting name for the role management permission
@@ -68,7 +65,6 @@ func GenerateBundlesDefaultRoles() []*settingsmsg.Bundle {
 		generateBundleUserRole(),
 		generateBundleGuestRole(),
 		generateBundleProfileRequest(),
-		generateBundleMetadataRole(),
 		generateBundleSpaceAdminRole(),
 	}
 }
@@ -434,36 +430,6 @@ func generateBundleProfileRequest() *settingsmsg.Bundle {
 	}
 }
 
-func generateBundleMetadataRole() *settingsmsg.Bundle {
-	return &settingsmsg.Bundle{
-		Id:          BundleUUIDRoleMetadata,
-		Name:        "metadata",
-		Type:        settingsmsg.Bundle_TYPE_ROLE,
-		Extension:   "ocis-roles",
-		DisplayName: "Metadata",
-		Resource: &settingsmsg.Resource{
-			Type: settingsmsg.Resource_TYPE_SYSTEM,
-		},
-		Settings: []*settingsmsg.Setting{
-			{
-				Id:          CreateSpacePermissionID,
-				Name:        CreateSpacePermissionName,
-				DisplayName: "Create own Space",
-				Description: "This permission allows to create a space owned by the current user.",
-				Resource: &settingsmsg.Resource{
-					Type: settingsmsg.Resource_TYPE_SYSTEM, // TODO resource type space? self? me? own?
-				},
-				Value: &settingsmsg.Setting_PermissionValue{
-					PermissionValue: &settingsmsg.Permission{
-						Operation:  settingsmsg.Permission_OPERATION_CREATE,
-						Constraint: settingsmsg.Permission_CONSTRAINT_OWN,
-					},
-				},
-			},
-		},
-	}
-}
-
 // TODO: languageSetting needed?
 var languageSetting = settingsmsg.Setting_SingleChoiceValue{
 	SingleChoiceValue: &settingsmsg.SingleChoiceList{
@@ -532,11 +498,6 @@ var languageSetting = settingsmsg.Setting_SingleChoiceValue{
 // DefaultRoleAssignments returns (as one might guess) the default role assignments
 func DefaultRoleAssignments() []*settingsmsg.UserRoleAssignment {
 	return []*settingsmsg.UserRoleAssignment{
-		// accounts service user for the metadata user is allowed to create spaces
-		{
-			AccountUuid: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			RoleId:      BundleUUIDRoleAdmin,
-		},
 		// default admin users
 		{
 			AccountUuid: "058bff95-6708-4fe5-91e4-9ea3d377588b",

extensions/settings/pkg/store/metadata/assignments.go

@@ -13,7 +13,7 @@ import (
 
 // ListRoleAssignments loads and returns all role assignments matching the given assignment identifier.
 func (s *Store) ListRoleAssignments(accountUUID string) ([]*settingsmsg.UserRoleAssignment, error) {
-	if s.mdc == nil || accountUUID == "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad" {
+	if s.mdc == nil {
 		return defaultRoleAssignments(accountUUID), nil
 	}
 	s.Init()

extensions/sharing/pkg/config/defaults/defaultconfig.go

@@ -48,9 +48,8 @@ func DefaultConfig() *config.Config {
 				JanitorRunInterval:         60,
 			},
 			CS3: config.UserSharingCS3Driver{
-				ProviderAddr:   "127.0.0.1:9215",
-				ServiceUserID:  "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-				ServiceUserIDP: "https://localhost:9200",
+				ProviderAddr:   "127.0.0.1:9215", // metadata storage
+				ServiceUserIDP: "internal",
 			},
 		},
 		PublicSharingDriver: "json",
@@ -69,9 +68,8 @@ func DefaultConfig() *config.Config {
 				JanitorRunInterval:         60,
 			},
 			CS3: config.PublicSharingCS3Driver{
-				ProviderAddr:   "127.0.0.1:9215",
-				ServiceUserID:  "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-				ServiceUserIDP: "https://localhost:9200",
+				ProviderAddr:   "127.0.0.1:9215", // metadata storage
+				ServiceUserIDP: "internal",
 			},
 		},
 		Events: config.Events{
@@ -125,9 +123,17 @@ func EnsureDefaults(cfg *config.Config) {
 		cfg.UserSharingDrivers.CS3.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey
 	}
 
+	if cfg.UserSharingDrivers.CS3.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" {
+		cfg.UserSharingDrivers.CS3.ServiceUserID = cfg.Commons.MetadataUserID
+	}
+
 	if cfg.PublicSharingDrivers.CS3.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" {
 		cfg.PublicSharingDrivers.CS3.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey
 	}
+
+	if cfg.PublicSharingDrivers.CS3.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" {
+		cfg.PublicSharingDrivers.CS3.ServiceUserID = cfg.Commons.MetadataUserID
+	}
 }
 
 func Sanitize(cfg *config.Config) {

extensions/sharing/pkg/config/parser/parse.go

@@ -42,9 +42,17 @@ func Validate(cfg *config.Config) error {
 		return shared.MissingMachineAuthApiKeyError(cfg.Service.Name)
 	}
 
+	if cfg.PublicSharingDriver == "cs3" && cfg.PublicSharingDrivers.CS3.ServiceUserID == "" {
+		return shared.MissingMetadataUserID(cfg.Service.Name)
+	}
+
 	if cfg.UserSharingDriver == "cs3" && cfg.UserSharingDrivers.CS3.MachineAuthAPIKey == "" {
 		return shared.MissingMachineAuthApiKeyError(cfg.Service.Name)
 	}
 
+	if cfg.UserSharingDriver == "cs3" && cfg.UserSharingDrivers.CS3.ServiceUserID == "" {
+		return shared.MissingMetadataUserID(cfg.Service.Name)
+	}
+
 	return nil
 }

extensions/storage-metadata/pkg/command/command.go

@@ -7,18 +7,18 @@ import (
 	"os"
 	"path"
 
-	"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config/parser"
-	"github.com/owncloud/ocis/ocis-pkg/log"
-	"github.com/owncloud/ocis/ocis-pkg/sync"
-	"github.com/owncloud/ocis/ocis-pkg/tracing"
-
+	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	"github.com/cs3org/reva/v2/cmd/revad/runtime"
 	"github.com/gofrs/uuid"
 	"github.com/oklog/run"
 	"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config"
+	"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config/parser"
 	"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
 	"github.com/owncloud/ocis/extensions/storage/pkg/service/external"
 	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/log"
+	"github.com/owncloud/ocis/ocis-pkg/sync"
+	"github.com/owncloud/ocis/ocis-pkg/tracing"
 	"github.com/owncloud/ocis/ocis-pkg/version"
 	"github.com/thejerf/suture/v4"
 	"github.com/urfave/cli/v2"
@@ -140,10 +140,74 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"grpc": map[string]interface{}{
 			"network": cfg.GRPC.Protocol,
 			"address": cfg.GRPC.Addr,
-			"interceptors": map[string]interface{}{
-				"log": map[string]interface{}{},
-			},
 			"services": map[string]interface{}{
+				"gateway": map[string]interface{}{
+					// registries are located on the gateway
+					"authregistrysvc":    cfg.GRPC.Addr,
+					"storageregistrysvc": cfg.GRPC.Addr,
+					// user metadata is located on the users services
+					"userprovidersvc":  cfg.GRPC.Addr,
+					"groupprovidersvc": cfg.GRPC.Addr,
+					"permissionssvc":   cfg.GRPC.Addr,
+					// other
+					"disable_home_creation_on_login": true, // metadata manually creates a space
+					// metadata always uses the simple upload, so no transfer secret or datagateway needed
+				},
+				"userprovider": map[string]interface{}{
+					"driver": "memory",
+					"drivers": map[string]interface{}{
+						"memory": map[string]interface{}{
+							"users": map[string]interface{}{
+								"serviceuser": map[string]interface{}{
+									"id": map[string]interface{}{
+										"opaqueId": cfg.MetadataUserID,
+										"idp":      "internal",
+										"type":     userpb.UserType_USER_TYPE_PRIMARY,
+									},
+									"username":     "serviceuser",
+									"display_name": "System User",
+								},
+							},
+						},
+					},
+				},
+				"authregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"machine": cfg.GRPC.Addr,
+							},
+						},
+					},
+				},
+				"authprovider": map[string]interface{}{
+					"auth_manager": "machine",
+					"auth_managers": map[string]interface{}{
+						"machine": map[string]interface{}{
+							"api_key":      cfg.MachineAuthAPIKey,
+							"gateway_addr": cfg.GRPC.Addr,
+						},
+					},
+				},
+				"permissions": map[string]interface{}{
+					"driver": "demo",
+					"drivers": map[string]interface{}{
+						"demo": map[string]interface{}{},
+					},
+				},
+				"storageregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"/": map[string]interface{}{
+									"address": cfg.GRPC.Addr,
+								},
+							},
+						},
+					},
+				},
 				"storageprovider": map[string]interface{}{
 					"driver":          cfg.Driver,
 					"drivers":         config.MetadataDrivers(cfg),
@@ -155,7 +219,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"http": map[string]interface{}{
 			"network": cfg.HTTP.Protocol,
 			"address": cfg.HTTP.Addr,
-			// TODO build services dynamically
+			// no datagateway needed as the metadata clients directly talk to the dataprovider with the simple protocol
 			"services": map[string]interface{}{
 				"dataprovider": map[string]interface{}{
 					"prefix":      "data",

extensions/storage-metadata/pkg/config/config.go

@@ -19,8 +19,10 @@ type Config struct {
 
 	Context context.Context `yaml:"context"`
 
-	TokenManager *TokenManager `yaml:"token_manager"`
-	Reva         *Reva         `yaml:"reva"`
+	TokenManager      *TokenManager `yaml:"token_manager"`
+	Reva              *Reva         `yaml:"reva"`
+	MachineAuthAPIKey string        `yaml:"machine_auth_api_key" env:"STORAGE_METADATA_MACHINE_AUTH_API_KEY"`
+	MetadataUserID    string        `yaml:"metadata_user_id"`
 
 	SkipUserGroupsInToken bool    `yaml:"skip_user_groups_in_token"`
 	Driver                string  `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"`
@@ -60,8 +62,8 @@ type GRPCConfig struct {
 }
 
 type HTTPConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_METADATA_HTTP_ADDR" desc:"The address of the http service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_HTTP_PROTOCOL" desc:"The transport protocol of the http service."`
 }
 
 type Drivers struct {