change default paths for oCIS services

.make/release.mk

@@ -13,8 +13,10 @@ release-dirs:
 release-linux: $(GOX) release-dirs
 	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-$(OUTPUT)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
 
+# docker specific packaging flags
+DOCKER_LDFLAGS += -X "$(OCIS_REPO)/ocis-pkg/config/defaults.BaseDataPathType=path" -X "$(OCIS_REPO)/ocis-pkg/config/defaults.BaseDataPathValue=/var/lib/ocis"
 release-linux-docker: $(GOX) release-dirs
-	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
+	$(GOX) -tags 'netgo $(TAGS)' -ldflags '-extldflags "-static" $(LDFLAGS) $(DOCKER_LDFLAGS)' -os 'linux' -arch 'amd64 386 arm64 arm' -output '$(DIST)/binaries/$(EXECUTABLE)-{{.OS}}-{{.Arch}}' ./cmd/$(NAME)
 
 .PHONY: release-windows
 release-windows: $(GOX) release-dirs

accounts/pkg/config/config.go

@@ -68,8 +68,9 @@ type Log struct {
 
 // Repo defines which storage implementation is to be used.
 type Repo struct {
-	Disk Disk
-	CS3  CS3
+	Backend string
+	Disk    Disk
+	CS3     CS3
 }
 
 // Disk is the local disk implementation of the storage.

accounts/pkg/flagset/flagset.go

@@ -1,8 +1,11 @@
 package flagset
 
 import (
+	"path"
+
 	"github.com/owncloud/ocis/accounts/pkg/config"
 	accounts "github.com/owncloud/ocis/accounts/pkg/proto/v0"
+	"github.com/owncloud/ocis/ocis-pkg/config/defaults"
 	"github.com/owncloud/ocis/ocis-pkg/flags"
 	"github.com/urfave/cli/v2"
 )
@@ -155,10 +158,17 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"ACCOUNTS_JWT_SECRET", "OCIS_JWT_SECRET"},
 			Destination: &cfg.TokenManager.JWTSecret,
 		},
+		&cli.StringFlag{
+			Name:        "storage-backend",
+			Value:       flags.OverrideDefaultString(cfg.Repo.Disk.Path, "CS3"),
+			Usage:       "Which backend to use to store accounts data (CS3 or disk)",
+			EnvVars:     []string{"ACCOUNTS_STORAGE_BACKEND"},
+			Destination: &cfg.Repo.Backend,
+		},
 		&cli.StringFlag{
 			Name:        "storage-disk-path",
-			Value:       flags.OverrideDefaultString(cfg.Repo.Disk.Path, ""),
-			Usage:       "Path on the local disk, e.g. /var/tmp/ocis/accounts",
+			Value:       flags.OverrideDefaultString(cfg.Repo.Disk.Path, path.Join(defaults.BaseDataPath(), "accounts")),
+			Usage:       "Path on the local disk to store accounts data when backend is set to disk",
 			EnvVars:     []string{"ACCOUNTS_STORAGE_DISK_PATH"},
 			Destination: &cfg.Repo.Disk.Path,
 		},

accounts/pkg/proto/v0/accounts.pb.micro_test.go

@@ -392,7 +392,7 @@ func deleteGroup(t *testing.T, id string) (*empty.Empty, error) {
 
 // createTmpDir creates a temporary dir for tests data.
 func createTmpDir() string {
-	name, err := ioutil.TempDir("/var/tmp", "ocis-accounts-store-")
+	name, err := ioutil.TempDir("/tmp", "ocis-accounts-store-")
 	if err != nil {
 		panic(err)
 	}

accounts/pkg/service/v0/accounts_permission_test.go

@@ -23,7 +23,7 @@ import (
 	"go-micro.dev/v4/metadata"
 )
 
-const dataPath = "/var/tmp/ocis-accounts-tests"
+const dataPath = "/tmp/ocis-accounts-tests"
 
 var (
 	roleServiceMock settings.RoleService

accounts/pkg/service/v0/service.go

@@ -58,7 +58,7 @@ func New(opts ...Option) (s *Service, err error) {
 	}
 
 	r := oreg.GetRegistry()
-	if cfg.Repo.Disk.Path == "" {
+	if strings.ToLower(cfg.Repo.Backend) != "disk" {
 		if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil {
 			logger.Error().Err(err).Msg("index: storage-metadata service not present")
 			return nil, err

deployments/examples/cs3_users_ocis/docker-compose.yml

@@ -89,7 +89,7 @@ services:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
       - ./config/ocis/web-config.dist.json:/config/web-config.dist.json
       - ./config/ocis/proxy-config.json:/config/proxy-config.json
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

deployments/examples/oc10_ocis_parallel/docker-compose.yml

@@ -119,7 +119,7 @@ services:
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
       - ./config/ocis/proxy-config.dist.json:/config/proxy-config.dist.json
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
       # shared volume with oC10
       - oc10-data:/mnt/data
     labels:

deployments/examples/ocis_hello/docker-compose.yml

@@ -69,7 +69,7 @@ services:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
       - ./config/ocis/web-config.dist.json:/config/web-config.dist.json
       - ./config/ocis/proxy-config.json:/config/proxy-config.json
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

deployments/examples/ocis_keycloak/docker-compose.yml

@@ -73,7 +73,7 @@ services:
       OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

deployments/examples/ocis_s3/docker-compose.yml

@@ -72,7 +72,7 @@ services:
       STORAGE_USERS_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

deployments/examples/ocis_traefik/docker-compose.yml

@@ -62,7 +62,7 @@ services:
       OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

deployments/examples/ocis_wopi/docker-compose.yml

@@ -72,7 +72,7 @@ services:
     volumes:
       - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
       - ./config/ocis/mimetypes.json:/var/tmp/ocis/app-config/mimetypes.json
-      - ocis-data:/var/tmp/ocis
+      - ocis-data:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"

docs/extensions/storage/storages.md

@@ -120,7 +120,7 @@ To provide the other storage aspects we plan to implement a FUSE overlay filesys
 This is the current default storage driver. While it implements the file tree (using redis, including id based lookup), ETag propagation, trash, versions and sharing (including expiry) using the data directory layout of ownCloud 10 it has [known limitations](https://github.com/owncloud/core/issues/28095) that cannot be fixed without changing the actual layout on disk.
 
 To setup it up properly in a distributed fashion, the storage-home and the storage-oc need to share the same underlying FS. Their "data" counterparts also need access to the same shared FS.
-For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder inside the containers.
+For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/lib/ocis` folder inside the containers.
 
 - tree provided by a POSIX filesystem
   - file layout is mapped to the old ownCloud 10 layout

docs/ocis/getting-started/_index.md

@@ -46,7 +46,7 @@ chmod +x ocis
 ./ocis server
 ```
 
-The default primary storage location is `/var/tmp/ocis`. You can change that value by configuration.
+The default primary storage location is `~/.ocis` or `/var/lib/ocis` depending on the packaging format and your operating system user. You can change that value by configuration.
 
 {{< hint warning >}}
 oCIS by default relies on Multicast DNS (mDNS), usually via avahi-daemon. If your system has a firewall, make sure mDNS is allowed in your active zone.

glauth/pkg/flagset/flagset.go

@@ -4,8 +4,8 @@ import (
 	"path"
 
 	"github.com/owncloud/ocis/glauth/pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/config/defaults"
 	"github.com/owncloud/ocis/ocis-pkg/flags"
-	pkgos "github.com/owncloud/ocis/ocis-pkg/os"
 	"github.com/urfave/cli/v2"
 )
 
@@ -148,27 +148,27 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 		&cli.StringFlag{
 			Name:        "ldaps-addr",
 			Value:       flags.OverrideDefaultString(cfg.Ldaps.Addr, "127.0.0.1:9126"),
-			Usage:       "Address to bind ldap server",
+			Usage:       "Address to bind ldaps server",
 			EnvVars:     []string{"GLAUTH_LDAPS_ADDR"},
 			Destination: &cfg.Ldaps.Addr,
 		},
 		&cli.BoolFlag{
 			Name:        "ldaps-enabled",
 			Value:       flags.OverrideDefaultBool(cfg.Ldaps.Enabled, true),
-			Usage:       "Enable ldap server",
+			Usage:       "Enable ldaps server",
 			EnvVars:     []string{"GLAUTH_LDAPS_ENABLED"},
 			Destination: &cfg.Ldaps.Enabled,
 		},
 		&cli.StringFlag{
 			Name:        "ldaps-cert",
-			Value:       flags.OverrideDefaultString(cfg.Ldaps.Cert, path.Join(pkgos.MustUserConfigDir("ocis", "ldap"), "ldap.crt")),
+			Value:       flags.OverrideDefaultString(cfg.Ldaps.Cert, path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt")),
 			Usage:       "path to ldaps certificate in PEM format",
 			EnvVars:     []string{"GLAUTH_LDAPS_CERT"},
 			Destination: &cfg.Ldaps.Cert,
 		},
 		&cli.StringFlag{
 			Name:        "ldaps-key",
-			Value:       flags.OverrideDefaultString(cfg.Ldaps.Key, path.Join(pkgos.MustUserConfigDir("ocis", "ldap"), "ldap.key")),
+			Value:       flags.OverrideDefaultString(cfg.Ldaps.Key, path.Join(defaults.BaseDataPath(), "ldap", "ldap.key")),
 			Usage:       "path to ldaps key in PEM format",
 			EnvVars:     []string{"GLAUTH_LDAPS_KEY"},
 			Destination: &cfg.Ldaps.Key,

idp/pkg/flagset/flagset.go

@@ -4,8 +4,8 @@ import (
 	"path"
 
 	"github.com/owncloud/ocis/idp/pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/config/defaults"
 	"github.com/owncloud/ocis/ocis-pkg/flags"
-	pkgos "github.com/owncloud/ocis/ocis-pkg/os"
 	"github.com/urfave/cli/v2"
 )
 
@@ -236,14 +236,14 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "transport-tls-cert",
-			Value:       flags.OverrideDefaultString(cfg.HTTP.TLSCert, path.Join(pkgos.MustUserConfigDir("ocis", "idp"), "server.crt")),
+			Value:       flags.OverrideDefaultString(cfg.HTTP.TLSCert, path.Join(defaults.BaseDataPath(), "idp", "server.crt")),
 			Usage:       "Certificate file for transport encryption",
 			EnvVars:     []string{"IDP_TRANSPORT_TLS_CERT"},
 			Destination: &cfg.HTTP.TLSCert,
 		},
 		&cli.StringFlag{
 			Name:        "transport-tls-key",
-			Value:       flags.OverrideDefaultString(cfg.HTTP.TLSKey, path.Join(pkgos.MustUserConfigDir("ocis", "idp"), "server.key")),
+			Value:       flags.OverrideDefaultString(cfg.HTTP.TLSKey, path.Join(defaults.BaseDataPath(), "idp", "server.key")),
 			Usage:       "Secret file for transport encryption",
 			EnvVars:     []string{"IDP_TRANSPORT_TLS_KEY"},
 			Destination: &cfg.HTTP.TLSKey,
@@ -335,14 +335,14 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 			Name:        "identifier-client-path",
 			Usage:       "Path to the identifier web client base folder",
 			EnvVars:     []string{"IDP_IDENTIFIER_CLIENT_PATH"},
-			Value:       flags.OverrideDefaultString(cfg.IDP.IdentifierClientPath, "/var/tmp/ocis/idp"),
+			Value:       flags.OverrideDefaultString(cfg.IDP.IdentifierClientPath, path.Join(defaults.BaseDataPath(), "idp")),
 			Destination: &cfg.IDP.IdentifierClientPath,
 		},
 		&cli.StringFlag{
 			Name:        "identifier-registration-conf",
 			Usage:       "Path to a identifier-registration.yaml configuration file",
 			EnvVars:     []string{"IDP_IDENTIFIER_REGISTRATION_CONF"},
-			Value:       flags.OverrideDefaultString(cfg.IDP.IdentifierRegistrationConf, "./config/identifier-registration.yaml"),
+			Value:       flags.OverrideDefaultString(cfg.IDP.IdentifierRegistrationConf, path.Join(defaults.BaseDataPath(), "idp", "identifier-registration.yaml")),
 			Destination: &cfg.IDP.IdentifierRegistrationConf,
 		},
 		&cli.StringFlag{

idp/pkg/server/http/server.go

@@ -18,20 +18,15 @@ func Server(opts ...Option) (http.Service, error) {
 
 	var tlsConfig *tls.Config
 	if options.Config.HTTP.TLS {
-		if options.Config.HTTP.TLSCert == "" || options.Config.HTTP.TLSKey == "" {
-			_, certErr := os.Stat("./server.crt")
-			_, keyErr := os.Stat("./server.key")
+		_, certErr := os.Stat(options.Config.HTTP.TLSCert)
+		_, keyErr := os.Stat(options.Config.HTTP.TLSKey)
 
-			if os.IsNotExist(certErr) || os.IsNotExist(keyErr) {
-				options.Logger.Info().Msgf("Generating certs")
-				if err := pkgcrypto.GenCert(options.Config.HTTP.TLSCert, options.Config.HTTP.TLSKey, options.Logger); err != nil {
-					options.Logger.Fatal().Err(err).Msg("Could not setup TLS")
-					os.Exit(1)
-				}
+		if os.IsNotExist(certErr) || os.IsNotExist(keyErr) {
+			options.Logger.Info().Msgf("Generating certs")
+			if err := pkgcrypto.GenCert(options.Config.HTTP.TLSCert, options.Config.HTTP.TLSKey, options.Logger); err != nil {
+				options.Logger.Fatal().Err(err).Msg("Could not setup TLS")
+				os.Exit(1)
 			}
-
-			options.Config.HTTP.TLSCert = "server.crt"
-			options.Config.HTTP.TLSKey = "server.key"
 		}
 
 		cer, err := tls.LoadX509KeyPair(options.Config.HTTP.TLSCert, options.Config.HTTP.TLSKey)

ocis-pkg/.gitignore

@@ -0,0 +1 @@
+!config

ocis-pkg/config/defaults/paths.go

@@ -0,0 +1,42 @@
+package defaults
+
+import (
+	"log"
+	"os"
+	"path"
+)
+
+const ()
+
+var (
+	// switch between modes
+	BaseDataPathType string = "homedir"
+	// don't read from this, only write
+	BaseDataPathValue string = "/var/lib/ocis"
+)
+
+func BaseDataPath() string {
+
+	// It is not nice to have hidden / secrete configuration options
+	// But how can we update the base path for every occurence with a flageset option?
+	// This is currenlty not possible and needs a new configuration concept
+	p := os.Getenv("OCIS_BASE_DATA_PATH")
+	if p != "" {
+		return p
+	}
+
+	switch BaseDataPathType {
+	case "homedir":
+		dir, err := os.UserHomeDir()
+		if err != nil {
+			// fallback to BaseDatapathValue for users without home
+			return BaseDataPathValue
+		}
+		return path.Join(dir, ".ocis")
+	case "path":
+		return BaseDataPathValue
+	default:
+		log.Fatalf("BaseDataPathType %s not found", BaseDataPathType)
+		return ""
+	}
+}

ocis-pkg/indexer/index/cs3/autoincrement_test.go

@@ -11,7 +11,7 @@ package cs3
 //	"github.com/stretchr/testify/assert"
 //)
 //
-//const cs3RootFolder = "/var/tmp/ocis/storage/users/data"
+//const cs3RootFolder = "/tmp/ocis/storage/users/data"
 //
 //func TestAutoincrementIndexAdd(t *testing.T) {
 //	dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)

ocis-pkg/indexer/index/cs3/non_unique.go

@@ -46,14 +46,14 @@ type NonUnique struct {
 }
 
 // NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance.
-// /var/tmp/ocis/accounts/index.cs3/Pets/Bro*
+// /tmp/ocis/accounts/index.cs3/Pets/Bro*
 // ├── Brown/
-// │   └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123
+// │   └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123
 // ├── Green/
-// │    ├── goefe-789 -> /var/tmp/testfiles-395764020/pets/goefe-789
-// │    └── xadaf-189 -> /var/tmp/testfiles-395764020/pets/xadaf-189
+// │    ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789
+// │    └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189
 // └── White/
-//     └── wefwe-456 -> /var/tmp/testfiles-395764020/pets/wefwe-456
+//     └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456
 func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
 	opts := &option.Options{}
 	for _, opt := range o {

ocis-pkg/indexer/index/disk/autoincrement_test.go

@@ -290,7 +290,7 @@ func BenchmarkAdd(b *testing.B) {
 }
 
 func createTmpDirStr() (string, error) {
-	name, err := ioutil.TempDir("/var/tmp", "testfiles-*")
+	name, err := ioutil.TempDir("/tmp", "testfiles-*")
 	if err != nil {
 		return "", err
 	}

ocis-pkg/indexer/index/disk/non_unique.go

@@ -16,14 +16,14 @@ import (
 
 // NonUnique is able to index an document by a key which might contain non-unique values
 //
-// /var/tmp/testfiles-395764020/index.disk/PetByColor/
+// /tmp/testfiles-395764020/index.disk/PetByColor/
 // ├── Brown
-// │   └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123
+// │   └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123
 // ├── Green
-// │    ├── goefe-789 -> /var/tmp/testfiles-395764020/pets/goefe-789
-// │    └── xadaf-189 -> /var/tmp/testfiles-395764020/pets/xadaf-189
+// │    ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789
+// │    └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189
 // └── White
-//     └── wefwe-456 -> /var/tmp/testfiles-395764020/pets/wefwe-456
+//     └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456
 type NonUnique struct {
 	caseInsensitive bool
 	indexBy         string

ocis-pkg/indexer/indexer_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-//const cs3RootFolder = "/var/tmp/ocis/storage/users/data"
+//const cs3RootFolder = "/tmp/ocis/storage/users/data"
 //
 //func TestIndexer_CS3_AddWithUniqueIndex(t *testing.T) {
 //	dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)

ocis-pkg/indexer/test/helpers.go

@@ -10,7 +10,7 @@ import (
 
 // CreateTmpDir creates a temporary dir for tests data.
 func CreateTmpDir() (string, error) {
-	name, err := ioutil.TempDir("/var/tmp", "testfiles-")
+	name, err := ioutil.TempDir("/tmp", "testfiles-")
 	if err != nil {
 		return "", err
 	}