introduce metadata gateway

.drone.star

@@ -1710,7 +1710,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "PROXY_ENABLE_BASIC_AUTH": True,
             "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
             "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
-            "OCIS_LOG_LEVEL": "error",
+            "OCIS_LOG_LEVEL": "debug",
             "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
             "OCIS_INSECURE": "true",
             "IDM_CREATE_DEMO_USERS": True,

changelog/unreleased/metadata-gateway.md

@@ -0,0 +1,5 @@
+Enhancement: wrap metadata storage with dedicated reva gateway
+
+We wrapped the metadata storage in a minimal reva instance with a dedicated gateway, including static storage registry, static auth registry, in memory userprovider, machine authprovider and demo permissions service. This allows us to preconfigure the service user for the ocis settings service, share and public share providers.
+
+https://github.com/owncloud/ocis/pull/3602

changelog/unreleased/update-reva.md

@@ -7,5 +7,6 @@ Updated reva to version 2.x.x. This update includes:
 https://github.com/owncloud/ocis/pull/3552
 https://github.com/owncloud/ocis/pull/3570
 https://github.com/owncloud/ocis/pull/3601
+https://github.com/owncloud/ocis/pull/3602
 https://github.com/owncloud/ocis/pull/3605
 https://github.com/owncloud/ocis/pull/3611

extensions/settings/pkg/config/defaults/defaultconfig.go

@@ -55,10 +55,10 @@ func DefaultConfig() *config.Config {
 		},
 
 		Metadata: config.Metadata{
-			GatewayAddress:    "127.0.0.1:9142",
+			GatewayAddress:    "127.0.0.1:9215",
 			StorageAddress:    "127.0.0.1:9215",
 			ServiceUserID:     "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			ServiceUserIDP:    "https://localhost:9200",
+			ServiceUserIDP:    "internal",
 			MachineAuthAPIKey: "change-me-please",
 		},
 	}

extensions/storage-metadata/pkg/command/command.go

@@ -125,16 +125,80 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		},
 		"shared": map[string]interface{}{
 			"jwt_secret":                cfg.JWTSecret,
-			"gatewaysvc":                cfg.GatewayEndpoint,
+			"gatewaysvc":                cfg.GRPC.Addr,
 			"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
 		},
 		"grpc": map[string]interface{}{
 			"network": cfg.GRPC.Protocol,
 			"address": cfg.GRPC.Addr,
-			"interceptors": map[string]interface{}{
-				"log": map[string]interface{}{},
-			},
 			"services": map[string]interface{}{
+				"gateway": map[string]interface{}{
+					// registries are located on the gateway
+					"authregistrysvc":    cfg.GRPC.Addr,
+					"storageregistrysvc": cfg.GRPC.Addr,
+					// user metadata is located on the users services
+					"userprovidersvc":  cfg.GRPC.Addr,
+					"groupprovidersvc": cfg.GRPC.Addr,
+					"permissionssvc":   cfg.GRPC.Addr,
+					// other
+					"disable_home_creation_on_login": true, // metadata manually creates a space
+					// metadata always uses the simple upload, so no transfer secret or datagateway needed
+				},
+				"userprovider": map[string]interface{}{
+					"driver": "memory",
+					"drivers": map[string]interface{}{
+						"memory": map[string]interface{}{
+							"users": map[string]interface{}{
+								"serviceuser": map[string]interface{}{
+									"id": map[string]interface{}{
+										"opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
+										"idp":      "internal",
+										"type":     1, // user.UserType_USER_TYPE_PRIMARY
+									},
+									"username":     "serviceuser",
+									"display_name": "System User",
+								},
+							},
+						},
+					},
+				},
+				"authregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"machine": cfg.GRPC.Addr,
+							},
+						},
+					},
+				},
+				"authprovider": map[string]interface{}{
+					"auth_manager": "machine",
+					"auth_managers": map[string]interface{}{
+						"machine": map[string]interface{}{
+							"api_key":      cfg.MachineAuthAPIKey,
+							"gateway_addr": cfg.GRPC.Addr,
+						},
+					},
+				},
+				"permissions": map[string]interface{}{
+					"driver": "demo",
+					"drivers": map[string]interface{}{
+						"demo": map[string]interface{}{},
+					},
+				},
+				"storageregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"/": map[string]interface{}{
+									"address": cfg.GRPC.Addr,
+								},
+							},
+						},
+					},
+				},
 				"storageprovider": map[string]interface{}{
 					"driver":          cfg.Driver,
 					"drivers":         config.MetadataDrivers(cfg),
@@ -146,7 +210,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"http": map[string]interface{}{
 			"network": cfg.HTTP.Protocol,
 			"address": cfg.HTTP.Addr,
-			// TODO build services dynamically
+			// no datagateway needed as the metadata clients directly talk to the dataprovider with the simple protocol
 			"services": map[string]interface{}{
 				"dataprovider": map[string]interface{}{
 					"prefix":      "data",

extensions/storage-metadata/pkg/config/config.go

@@ -19,13 +19,13 @@ type Config struct {
 
 	Context               context.Context
 	JWTSecret             string
-	GatewayEndpoint       string
 	SkipUserGroupsInToken bool
 	Driver                string  `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"`
 	Drivers               Drivers `yaml:"drivers"`
 	DataServerURL         string
 	TempFolder            string
-	DataProviderInsecure  bool `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"`
+	DataProviderInsecure  bool   `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"`
+	MachineAuthAPIKey     string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"`
 }
 type Tracing struct {
 	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
@@ -58,8 +58,8 @@ type GRPCConfig struct {
 }
 
 type HTTPConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_METADATA_HTTP_ADDR" desc:"The address of the http service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_HTTP_PROTOCOL" desc:"The transport protocol of the http service."`
 }
 
 type Drivers struct {

extensions/storage-metadata/pkg/config/defaults/defaultconfig.go

@@ -35,11 +35,11 @@ func DefaultConfig() *config.Config {
 		Service: config.Service{
 			Name: "storage-metadata",
 		},
-		GatewayEndpoint: "127.0.0.1:9142",
-		JWTSecret:       "Pive-Fumkiu4",
-		TempFolder:      filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"),
-		DataServerURL:   "http://localhost:9216/data",
-		Driver:          "ocis",
+		JWTSecret:         "Pive-Fumkiu4",
+		TempFolder:        filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"),
+		DataServerURL:     "http://localhost:9216/data",
+		MachineAuthAPIKey: "change-me-please",
+		Driver:            "ocis",
 		Drivers: config.Drivers{
 			EOS: config.EOSDriver{
 				Root:                "/eos/dockertest/reva",
@@ -59,7 +59,7 @@ func DefaultConfig() *config.Config {
 				SecProtocol:         "",
 				Keytab:              "",
 				SingleUsername:      "",
-				GatewaySVC:          "127.0.0.1:9142",
+				GatewaySVC:          "127.0.0.1:9215",
 			},
 			Local: config.LocalDriver{
 				Root: filepath.Join(defaults.BaseDataPath(), "storage", "local", "metadata"),
@@ -71,12 +71,12 @@ func DefaultConfig() *config.Config {
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
 				Region:              "default",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 			OCIS: config.OCISDriver{
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 		},
 	}

extensions/storage-publiclink/pkg/config/config.go

@@ -24,33 +24,33 @@ type Config struct {
 	StorageProvider       StorageProvider
 }
 type Tracing struct {
-	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
-	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
-	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
-	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
+	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_PUBLICLINK_TRACING_ENABLED" desc:"Activates tracing."`
+	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_PUBLICLINK_TRACING_TYPE"`
+	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_PUBLICLINK_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
+	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_PUBLICLINK_TRACING_COLLECTOR"`
 }
 
 type Logging struct {
-	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
-	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
-	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
-	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
+	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_PUBLICLINK_LOG_LEVEL" desc:"The log level."`
+	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_PUBLICLINK_LOG_PRETTY" desc:"Activates pretty log output."`
+	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_PUBLICLINK_LOG_COLOR" desc:"Activates colorized log output."`
+	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_PUBLICLINK_LOG_FILE" desc:"The target log file."`
 }
 
 type Service struct {
 	Name string `yaml:"-"`
 }
 
 type Debug struct {
-	Addr   string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
-	Token  string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
-	Pprof  bool   `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
-	Zpages bool   `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
+	Addr   string `yaml:"addr" env:"STORAGE_PUBLICLINK_DEBUG_ADDR"`
+	Token  string `yaml:"token" env:"STORAGE_PUBLICLINK_DEBUG_TOKEN"`
+	Pprof  bool   `yaml:"pprof" env:"STORAGE_PUBLICLINK_DEBUG_PPROF"`
+	Zpages bool   `yaml:"zpages" env:"STORAGE_PUBLICLINK_DEBUG_ZPAGES"`
 }
 
 type GRPCConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_PUBLICLINK_GRPC_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_PUBLICLINK_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }
 
 type AuthProvider struct {

extensions/storage-shares/pkg/config/config.go

@@ -25,36 +25,36 @@ type Config struct {
 	SharesProviderEndpoint string
 }
 type Tracing struct {
-	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
-	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
-	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
-	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
+	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_SHARES_TRACING_ENABLED" desc:"Activates tracing."`
+	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_SHARES_TRACING_TYPE"`
+	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_SHARES_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
+	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_SHARES_TRACING_COLLECTOR"`
 }
 
 type Logging struct {
-	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
-	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
-	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
-	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
+	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_SHARES_LOG_LEVEL" desc:"The log level."`
+	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_SHARES_LOG_PRETTY" desc:"Activates pretty log output."`
+	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_SHARES_LOG_COLOR" desc:"Activates colorized log output."`
+	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_SHARES_LOG_FILE" desc:"The target log file."`
 }
 
 type Service struct {
 	Name string `yaml:"-"`
 }
 
 type Debug struct {
-	Addr   string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
-	Token  string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
-	Pprof  bool   `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
-	Zpages bool   `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
+	Addr   string `yaml:"addr" env:"STORAGE_SHARES_DEBUG_ADDR"`
+	Token  string `yaml:"token" env:"STORAGE_SHARES_DEBUG_TOKEN"`
+	Pprof  bool   `yaml:"pprof" env:"STORAGE_SHARES_DEBUG_PPROF"`
+	Zpages bool   `yaml:"zpages" env:"STORAGE_SHARES_DEBUG_ZPAGES"`
 }
 
 type GRPCConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_SHARES_GRPC_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_SHARES_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }
 
 type HTTPConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_SHARES_HTTP_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_SHARES_HTTP_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }

extensions/storage/pkg/config/defaults/defaultconfig.go

@@ -339,9 +339,9 @@ func DefaultConfig() *config.Config {
 					GRPCAddr:    "127.0.0.1:9150",
 					Services:    []string{"usershareprovider", "publicshareprovider"},
 				},
-				CS3ProviderAddr:                  "127.0.0.1:9215",
+				CS3ProviderAddr:                  "127.0.0.1:9215", // metadata storage
 				CS3ServiceUser:                   "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-				CS3ServiceUserIdp:                "https://localhost:9200",
+				CS3ServiceUserIdp:                "internal",
 				UserDriver:                       "json",
 				UserJSONFile:                     path.Join(defaults.BaseDataPath(), "storage", "shares.json"),
 				UserSQLUsername:                  "",

go.sum

@@ -318,8 +318,6 @@ github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0
 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
 github.com/cs3org/reva v1.18.0 h1:MbPS5ZAa8RzKcTxAVeSDdISB3XXqLIxqB03BTN5ReBY=
 github.com/cs3org/reva v1.18.0/go.mod h1:e5VDUDu4vVWIeVkZcW//n6UZzhGGMa+Tz/whCiX3N6o=
-github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515 h1:8pPCLxNXVz/q7PMM6Zq1lff3P8SFAu8/CXwB2eA21xc=
-github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng=
 github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3 h1:6sKjGI0AUW5tBXWBduaBoc+9sNYZWQR894G0oFCbus0=
 github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=