diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index 8ff36871942..b58bcce5472 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -143,7 +143,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { &cli.StringFlag{ Name: "storage-disk-path", Value: "", - Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts", + Usage: "Path on the local disk, e.g. /var/tmp/ocis/accounts", EnvVars: []string{"ACCOUNTS_STORAGE_DISK_PATH"}, Destination: &cfg.Repo.Disk.Path, }, diff --git a/changelog/unreleased/make-all-paths-configurable.md b/changelog/unreleased/make-all-paths-configurable.md new file mode 100644 index 00000000000..f53008c206c --- /dev/null +++ b/changelog/unreleased/make-all-paths-configurable.md @@ -0,0 +1,5 @@ +Change: Make all paths configurable and default to a common temp dir + +Aligned all services to use a dir following`/var/tmp/ocis//...` by default. Also made some missing temp paths configurable via env vars and config flags. + +https://github.com/owncloud/ocis/pull/1080 diff --git a/docs/extensions/storage/storages.md b/docs/extensions/storage/storages.md index 2992dfda0e6..a05917a32fe 100644 --- a/docs/extensions/storage/storages.md +++ b/docs/extensions/storage/storages.md @@ -120,7 +120,7 @@ To provide the other storage aspects we plan to implement a FUSE overlay filesys This is the current default storage driver. While it implements the file tree (using redis, including id based lookup), ETag propagation, trash, versions and sharing (including expiry) using the data directory layout of ownCloud 10 it has [known limitations](https://github.com/owncloud/core/issues/28095) that cannot be fixed without changing the actual layout on disk. To setup it up properly in a distributed fashion, the storage-home and the storage-oc need to share the same underlying FS. Their "data" counterparts also need access to the same shared FS. -For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder insde the containers. +For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder inside the containers. - tree provided by a POSIX filesystem - file layout is mapped to the old ownCloud 10 layout diff --git a/docs/ocis/getting-started.md b/docs/ocis/getting-started.md index a61c1d8f111..2c68f85e8d5 100644 --- a/docs/ocis/getting-started.md +++ b/docs/ocis/getting-started.md @@ -29,7 +29,7 @@ chmod +x ocis ./ocis server ``` -The default primary storage location is `/var/tmp/`. You can change that value by configuration. +The default primary storage location is `/var/tmp/ocis`. You can change that value by configuration. ### Docker diff --git a/konnectd/pkg/flagset/flagset.go b/konnectd/pkg/flagset/flagset.go index 243d3e31660..0dce3529111 100644 --- a/konnectd/pkg/flagset/flagset.go +++ b/konnectd/pkg/flagset/flagset.go @@ -251,7 +251,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { Name: "identifier-client-path", Usage: "Path to the identifier web client base folder", EnvVars: []string{"KONNECTD_IDENTIFIER_CLIENT_PATH"}, - Value: "/var/tmp/konnectd", + Value: "/var/tmp/ocis/konnectd", Destination: &cfg.Konnectd.IdentifierClientPath, }, &cli.StringFlag{ diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index c93376891c6..fd0e5e63066 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -45,7 +45,7 @@ type NonUnique struct { } // NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance. -// /var/tmp/ocis-accounts/index.cs3/Pets/Bro* +// /var/tmp/ocis/accounts/index.cs3/Pets/Bro* // ├── Brown/ // │ └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123 // ├── Green/ diff --git a/ocis/docker-compose-eos-test.yml b/ocis/docker-compose-eos-test.yml index a8a341be0d3..f796d8a4ca5 100644 --- a/ocis/docker-compose-eos-test.yml +++ b/ocis/docker-compose-eos-test.yml @@ -44,7 +44,7 @@ services: # make accounts use ocis storage driver # TODO provision metadata storage in eos and switch to cs3 backend for accounts ACCOUNTS_LOG_LEVEL: debug - ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis-accounts + ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis/accounts # TODO make id the default in ocis-storage STORAGE_DRIVER_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}" STORAGE_FRONTEND_PUBLIC_URL: https://${OCIS_DOMAIN:-localhost}:9200 diff --git a/settings/pkg/flagset/flagset.go b/settings/pkg/flagset/flagset.go index ab3adfdc138..f4d04aba485 100644 --- a/settings/pkg/flagset/flagset.go +++ b/settings/pkg/flagset/flagset.go @@ -173,7 +173,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-settings", + Value: "/var/tmp/ocis/settings", Usage: "Mount path for the storage", EnvVars: []string{"SETTINGS_DATA_PATH"}, Destination: &cfg.Service.DataPath, diff --git a/settings/ui/tests/run-acceptance-test.sh b/settings/ui/tests/run-acceptance-test.sh index 939af07707b..53f732533bf 100755 --- a/settings/ui/tests/run-acceptance-test.sh +++ b/settings/ui/tests/run-acceptance-test.sh @@ -51,7 +51,7 @@ cp -r "$WEB_PATH/tests" "./$testFolder" export NODE_TLS_REJECT_UNAUTHORIZED='0' export SERVER_HOST=${SERVER_HOST:-https://localhost:9200} export BACKEND_HOST=${BACKEND_HOST:-https://localhost:9200} -export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis-settings"} +export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis/settings"} export RUN_ON_OCIS=true export TEST_TAGS=${TEST_TAGS:-"not @skip"} diff --git a/storage/pkg/command/frontend.go b/storage/pkg/command/frontend.go index 60455238f2a..7b122057b6b 100644 --- a/storage/pkg/command/frontend.go +++ b/storage/pkg/command/frontend.go @@ -130,7 +130,7 @@ func Frontend(cfg *config.Config) *cli.Command { }, "ocdav": map[string]interface{}{ "prefix": cfg.Reva.Frontend.OCDavPrefix, - "chunk_folder": "/var/tmp/ocis/chunks", + "chunk_folder": cfg.Reva.OCDav.ChunkFolder, "files_namespace": cfg.Reva.OCDav.DavFilesNamespace, "webdav_namespace": cfg.Reva.OCDav.WebdavNamespace, "timeout": 86400, diff --git a/storage/pkg/command/sharing.go b/storage/pkg/command/sharing.go index 4d48fb8e3a8..b8daaf1070c 100644 --- a/storage/pkg/command/sharing.go +++ b/storage/pkg/command/sharing.go @@ -97,6 +97,11 @@ func Sharing(cfg *config.Config) *cli.Command { }, "publicshareprovider": map[string]interface{}{ "driver": cfg.Reva.Sharing.PublicDriver, + "drivers": map[string]interface{}{ + "json": map[string]interface{}{ + "file": cfg.Reva.Sharing.PublicJSONFile, + }, + }, }, }, }, diff --git a/storage/pkg/command/storagehome.go b/storage/pkg/command/storagehome.go index 3e7cee19229..2815c6d598d 100644 --- a/storage/pkg/command/storagehome.go +++ b/storage/pkg/command/storagehome.go @@ -102,6 +102,7 @@ func StorageHome(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageHome.MountID, "expose_data_server": cfg.Reva.StorageHome.ExposeDataServer, "data_server_url": cfg.Reva.StorageHome.DataServerURL, + "tmp_folder": cfg.Reva.StorageHome.TempFolder, }, }, }, diff --git a/storage/pkg/command/storagemetadata.go b/storage/pkg/command/storagemetadata.go index c21277e75b0..51c5c790f92 100644 --- a/storage/pkg/command/storagemetadata.go +++ b/storage/pkg/command/storagemetadata.go @@ -111,6 +111,7 @@ func StorageMetadata(cfg *config.Config) *cli.Command { "driver": cfg.Reva.StorageMetadata.Driver, "drivers": drivers(cfg), "data_server_url": cfg.Reva.StorageMetadata.DataServerURL, + "tmp_folder": cfg.Reva.StorageMetadata.TempFolder, }, }, }, diff --git a/storage/pkg/command/storageusers.go b/storage/pkg/command/storageusers.go index 6e752adc0f8..17c9a93fb81 100644 --- a/storage/pkg/command/storageusers.go +++ b/storage/pkg/command/storageusers.go @@ -102,6 +102,7 @@ func StorageUsers(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageUsers.MountID, "expose_data_server": cfg.Reva.StorageUsers.ExposeDataServer, "data_server_url": cfg.Reva.StorageUsers.DataServerURL, + "tmp_folder": cfg.Reva.StorageUsers.TempFolder, }, }, }, diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index d60a166fa26..0a38f0f8fd8 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -36,9 +36,10 @@ type StorageRegistry struct { // Sharing defines the available sharing configuration. type Sharing struct { Port - UserDriver string - UserJSONFile string - PublicDriver string + UserDriver string + UserJSONFile string + PublicDriver string + PublicJSONFile string } // Port defines the available port configuration. @@ -82,7 +83,7 @@ type FrontendPort struct { DatagatewayPrefix string OCDavPrefix string OCSPrefix string - OCSSharePrefix string + OCSSharePrefix string PublicURL string Middleware Middleware } @@ -282,6 +283,7 @@ type LDAPSchema struct { // OCDav defines the available ocdav configuration. type OCDav struct { + ChunkFolder string WebdavNamespace string DavFilesNamespace string } diff --git a/storage/pkg/flagset/driverlocal.go b/storage/pkg/flagset/driverlocal.go index 6acd21030d3..6184fa2b38f 100644 --- a/storage/pkg/flagset/driverlocal.go +++ b/storage/pkg/flagset/driverlocal.go @@ -10,7 +10,7 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-local-root", - Value: "/var/tmp/ocis/local", + Value: "/var/tmp/ocis/storage/local", Usage: "the path to the local storage root", EnvVars: []string{"STORAGE_DRIVER_LOCAL_ROOT"}, Destination: &cfg.Reva.Storages.Local.Root, diff --git a/storage/pkg/flagset/driverowncloud.go b/storage/pkg/flagset/driverowncloud.go index c747291948f..633c0b4df69 100644 --- a/storage/pkg/flagset/driverowncloud.go +++ b/storage/pkg/flagset/driverowncloud.go @@ -10,14 +10,14 @@ func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-owncloud-datadir", - Value: "/var/tmp/ocis/owncloud", + Value: "/var/tmp/ocis/storage/owncloud", Usage: "the path to the owncloud data directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_DATADIR"}, Destination: &cfg.Reva.Storages.OwnCloud.Root, }, &cli.StringFlag{ Name: "storage-owncloud-uploadinfo-dir", - Value: "/var/tmp/ocis/uploadinfo", + Value: "/var/tmp/ocis/storage/uploadinfo", Usage: "the path to the tus upload info directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, Destination: &cfg.Reva.Storages.OwnCloud.UploadInfoDir, diff --git a/storage/pkg/flagset/frontend.go b/storage/pkg/flagset/frontend.go index cddbc5e5dc2..6cfaf820afb 100644 --- a/storage/pkg/flagset/frontend.go +++ b/storage/pkg/flagset/frontend.go @@ -30,11 +30,18 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { // OCDav + &cli.StringFlag{ + Name: "chunk-folder", + Value: "/var/tmp/ocis/tmp/chunks", + Usage: "temp directory for chunked uploads", + EnvVars: []string{"STORAGE_CHUNK_FOLDER"}, + Destination: &cfg.Reva.OCDav.WebdavNamespace, + }, &cli.StringFlag{ Name: "webdav-namespace", Value: "/home/", Usage: "Namespace prefix for the /webdav endpoint", - EnvVars: []string{"WEBDAV_NAMESPACE"}, + EnvVars: []string{"STORAGE_WEBDAV_NAMESPACE"}, Destination: &cfg.Reva.OCDav.WebdavNamespace, }, @@ -44,7 +51,7 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { Name: "dav-files-namespace", Value: "/users/", Usage: "Namespace prefix for the webdav /dav/files endpoint", - EnvVars: []string{"DAV_FILES_NAMESPACE"}, + EnvVars: []string{"STORAGE_DAV_FILES_NAMESPACE"}, Destination: &cfg.Reva.OCDav.DavFilesNamespace, }, diff --git a/storage/pkg/flagset/sharing.go b/storage/pkg/flagset/sharing.go index 3b6129f478e..fac4fc0cc39 100644 --- a/storage/pkg/flagset/sharing.go +++ b/storage/pkg/flagset/sharing.go @@ -51,7 +51,7 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "user-json-file", - Value: "/var/tmp/ocis/shares.json", + Value: "/var/tmp/ocis/storage/shares.json", Usage: "file used to persist shares for the UserShareProvider", EnvVars: []string{"STORAGE_SHARING_USER_JSON_FILE"}, Destination: &cfg.Reva.Sharing.UserJSONFile, @@ -63,6 +63,13 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_SHARING_PUBLIC_DRIVER"}, Destination: &cfg.Reva.Sharing.PublicDriver, }, + &cli.StringFlag{ + Name: "public-json-file", + Value: "/var/tmp/ocis/storage/publicshares.json", + Usage: "file used to persist shares for the PublicShareProvider", + EnvVars: []string{"STORAGE_SHARING_PUBLIC_JSON_FILE"}, + Destination: &cfg.Reva.Sharing.PublicJSONFile, + }, } flags = append(flags, TracingWithConfig(cfg)...) diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 72d0d0c5476..b6c0fe9fc42 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -111,6 +111,13 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_HOME_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageHome.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/home", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_HOME_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageHome.TempFolder, + }, &cli.BoolFlag{ Name: "enable-home", Value: true, diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index f0585c023c6..8119834474f 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -50,6 +50,13 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_METADATA_HTTP_ADDR"}, Destination: &cfg.Reva.StorageMetadata.HTTPAddr, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/metadata", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_METADATA_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageMetadata.TempFolder, + }, &cli.StringFlag{ Name: "driver", Value: "ocis", @@ -91,7 +98,7 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { flags = append(flags, &cli.StringFlag{ Name: "storage-root", - Value: "/var/tmp/ocis/metadata", + Value: "/var/tmp/ocis/storage/metadata", Usage: "the path to the metadata storage root", EnvVars: []string{"STORAGE_METADATA_ROOT"}, Destination: &cfg.Reva.Storages.Common.Root, diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index c5a53d198b5..4e6c3be7553 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -108,6 +108,13 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageUsers.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/users", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_USERS_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageUsers.TempFolder, + }, // some drivers need to look up users at the gateway diff --git a/store/pkg/flagset/flagset.go b/store/pkg/flagset/flagset.go index b731319da04..cfb8c7bd4e8 100644 --- a/store/pkg/flagset/flagset.go +++ b/store/pkg/flagset/flagset.go @@ -131,7 +131,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-store", + Value: "/var/tmp/ocis/store", Usage: "location of the store data path", EnvVars: []string{"STORE_DATA_PATH"}, Destination: &cfg.Datapath, diff --git a/store/pkg/service/v0/service.go b/store/pkg/service/v0/service.go index c773cc66c88..f8e8576a2ae 100644 --- a/store/pkg/service/v0/service.go +++ b/store/pkg/service/v0/service.go @@ -242,7 +242,7 @@ func (s *Service) Tables(ctx context.Context, in *proto.TablesRequest, out *prot } // TODO sanitize key. As it may contain invalid characters, such as slashes. -// file: /var/tmp/ocis-store/databases/{database}/{table}/{record.key}. +// file: /var/tmp/ocis/store/databases/{database}/{table}/{record.key}. func getID(database string, table string, key string) string { // TODO sanitize input. return filepath.Join(database, table, key)