From 800afdba3dd655ceeed5e7059df3e748bed9949b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 3 Aug 2021 11:02:44 +0200 Subject: [PATCH 01/16] Use field mask to update received shares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The next step for the shares storahge provider is to introduce field masks so the shares can update state and mount point. This PR is based on the full changeset, but removes everything but the API change. Add a sharesstorageprovider The provider exposes all received shares. It can be mounted to /home/Shares to make a lot of special cases for shares handling in the gateway and storage drivers superfluous. Add missing mock file Implement methods for setting/unsetting arbitrary metadata Fix tests Adapt to changes from rebase Fix creating references with embedded mounts Fix corner cases when stating shares Allow moves between shares on the same storage Make the storage rules known to the gateway as well Do not choke on non-existent shares Reject a share when it is being deleted Fix rebase artifacts WIP: Refactor statting shares. Merge shares permissions. update after rebase, fix tests Signed-off-by: Jörn Friedrich Dreyer list all shares Signed-off-by: Jörn Friedrich Dreyer work on api change Signed-off-by: Jörn Friedrich Dreyer Fix build Persist mountpoints in the share managers followin the new cs3 api Add support for renaming shares in the SharesStorageprovider Adapt commands for updating received shares Regenerate the share manager mock Fix linter warning Do not raise an internal error when trying to access non-existent shares Make hound happy Fix wrong column name in query Fix typo Do not confuse user and group names Add test for listing received group shares Do not list parent group shares if there is a child share for it already Make hound happy Hide the fact that accepted groups shares can be child shares in the db list shares using the shares manager + hide group shares when the same resource has a user and group share refactor all the ocs error writing from the new code Only collide with mountpoints of shares pointing do different resources Also return shares being shared with one of the user's groups Add sharesstorageprovider service file for local acceptance tests Adapt nextcloud share manager to new method signature Also remove the test for UpdateReceivedShare which can not be tested anymore with the new signature. The ReceivedShare never held the display name that's being tested so the test only passed on the data from the update field, but since the method only takes the actual received share now this is no longer possible. WIP: use go-cs3apis fork until it has been merged Add placeholder changelog to make CI run Tweak documentation on how to run the acceptance tests Add missing storage registry rule for the sharesstorageprovider Fix revad config for local acceptance tests reduce changes to share api change Signed-off-by: Jörn Friedrich Dreyer JSON encode the FieldMask parameter as-is paramsObj.Ref -> paramsObj.ReceivedShare Restore deleted test update code comment remove go mod replace Signed-off-by: Jörn Friedrich Dreyer --- go.mod | 2 +- go.sum | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3e97f71c06..1c41ab8198 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11 + github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 diff --git a/go.sum b/go.sum index 4fed162719..223e49d310 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,12 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11 h1:cc/8fdzWdr/wAZOXb29J8bnXjo1poCMCLwhlFBlvhfI= -github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535 h1:555D8A3ddKqb4OyK9v5mdphw2zDLWKGXOkcnf1RQwTA= +github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20210922150613-cb9e3c99f8de h1:N+AI8wz7yhDDqHDuq9EGaqQoFhAOi9XW37xt0ormflw= +github.com/cs3org/go-cs3apis v0.0.0-20210922150613-cb9e3c99f8de/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082 h1:ErxzuD05JkSlTQrqc8YSca7R1BPFuCesufg8gxzTB2g= +github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 764c8beb1c272a400570cfadad00e253b93bde55 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 6 Oct 2021 21:37:59 +0200 Subject: [PATCH 02/16] update cs3apis --- go.mod | 2 +- go.sum | 18 ++++++++++++------ .../grpc/services/gateway/storageprovider.go | 2 +- .../storageprovider/storageprovider.go | 2 +- pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/s3/s3.go | 2 +- pkg/storage/storage.go | 2 +- pkg/storage/utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/localfs/localfs_unix.go | 3 ++- 10 files changed, 22 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1c41ab8198..032a27b317 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082 + github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 diff --git a/go.sum b/go.sum index 223e49d310..9f762f9bbb 100644 --- a/go.sum +++ b/go.sum @@ -90,12 +90,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535 h1:555D8A3ddKqb4OyK9v5mdphw2zDLWKGXOkcnf1RQwTA= -github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20210922150613-cb9e3c99f8de h1:N+AI8wz7yhDDqHDuq9EGaqQoFhAOi9XW37xt0ormflw= -github.com/cs3org/go-cs3apis v0.0.0-20210922150613-cb9e3c99f8de/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082 h1:ErxzuD05JkSlTQrqc8YSca7R1BPFuCesufg8gxzTB2g= -github.com/cs3org/go-cs3apis v0.0.0-20211004093007-d29741980082/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30 h1:pbropYqSquVy4hWqKEhbVnITo0hJxdfl5Cew8zuiwkI= +github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -490,14 +486,24 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +<<<<<<< HEAD go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0 h1:Wx7nFnvCaissIUZxPkBqDz2963Z+Cl+PkYbDKzTxDqQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +======= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.24.0 h1:1hCzM7mwQbFQgk3Q4lAVEsGV6NB4Uj6Jt3EU+OiSBc8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.24.0/go.mod h1:O0cG0vP6TP3c323kh70JmeG1jN69Sn9Z5HxgmeASFWY= +go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +>>>>>>> update cs3apis go.opentelemetry.io/otel v1.0.1 h1:4XKyXmfqJLOQ7feyV5DB6gsBFZ0ltB8vLtp6pj4JIcc= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel/exporters/jaeger v1.0.1 h1:fg9udWIWWJMAT+Gq2ATFd/DFy3OZvKEZy9VK2amxvkw= go.opentelemetry.io/otel/exporters/jaeger v1.0.1/go.mod h1:85Ym3qknJdIdfRzYS9Ofy9NeLi9gKPFzFDBEHCKpfXI= go.opentelemetry.io/otel/sdk v1.0.1 h1:wXxFEWGo7XfXupPwVJvTBOaPBC9FEg0wB8hMNrKk+cA= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= +<<<<<<< HEAD +======= +go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= +>>>>>>> update cs3apis go.opentelemetry.io/otel/trace v1.0.1 h1:StTeIH6Q3G4r0Fiw34LTokUFESZgIDUr0qIJ7mKmAfw= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 249aac85e6..52a5999a65 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -2135,7 +2135,7 @@ func (s *svc) GetQuota(ctx context.Context, req *gateway.GetQuotaRequest) (*prov res, err := c.GetQuota(ctx, &provider.GetQuotaRequest{ Opaque: req.GetOpaque(), - // Ref: req.GetRef(), // TODO send which storage space ... or root + Ref: req.GetRef(), }) if err != nil { return nil, errors.Wrap(err, "gateway: error calling GetQuota") diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 0560c2278d..0aa25edc74 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -1181,7 +1181,7 @@ func (s *service) CreateSymlink(ctx context.Context, req *provider.CreateSymlink } func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (*provider.GetQuotaResponse, error) { - total, used, err := s.storage.GetQuota(ctx) + total, used, err := s.storage.GetQuota(ctx, req) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 43ac7f1fac..10d6af265b 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -679,7 +679,7 @@ func (nc *StorageDriver) ListGrants(ctx context.Context, ref *provider.Reference } // GetQuota as defined in the storage.FS interface -func (nc *StorageDriver) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (nc *StorageDriver) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { log := appctx.GetLogger(ctx) log.Info().Msg("GetQuota") diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index f1522deb11..7091d8514e 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -268,7 +268,7 @@ func (fs *s3FS) UpdateGrant(ctx context.Context, ref *provider.Reference, g *pro return errtypes.NotSupported("s3: operation not supported") } -func (fs *s3FS) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *s3FS) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { return 0, 0, nil } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index ff39b55b26..4179b4af02 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -52,7 +52,7 @@ type FS interface { RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) - GetQuota(ctx context.Context) ( /*TotalBytes*/ uint64 /*UsedBytes*/, uint64, error) + GetQuota(ctx context.Context, req *provider.GetQuotaRequest) ( /*TotalBytes*/ uint64 /*UsedBytes*/, uint64, error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error Shutdown(ctx context.Context) error SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index f57731c847..41d8cee8a1 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -128,7 +128,7 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error { // GetQuota returns the quota available // TODO Document in the cs3 should we return quota or free space? -func (fs *Decomposedfs) GetQuota(ctx context.Context) (total uint64, inUse uint64, err error) { +func (fs *Decomposedfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (total uint64, inUse uint64, err error) { var n *node.Node if n, err = fs.lu.HomeOrRootNode(ctx); err != nil { return 0, 0, err diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 909c0edbfb..4f522ad340 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -973,7 +973,7 @@ func (fs *eosfs) CreateStorageSpace(ctx context.Context, req *provider.CreateSto return nil, fmt.Errorf("unimplemented: CreateStorageSpace") } -func (fs *eosfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *eosfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { u, err := getUser(ctx) if err != nil { return 0, 0, errors.Wrap(err, "eosfs: no user in ctx") diff --git a/pkg/storage/utils/localfs/localfs_unix.go b/pkg/storage/utils/localfs/localfs_unix.go index 29ea2224af..6f9b8aa61c 100644 --- a/pkg/storage/utils/localfs/localfs_unix.go +++ b/pkg/storage/utils/localfs/localfs_unix.go @@ -30,6 +30,7 @@ import ( "strings" "syscall" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ) @@ -66,7 +67,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *localfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *localfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root From 5e219fb5b1bb61052a9a4e0141bf20c16561f562 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Fri, 8 Oct 2021 08:45:11 +0200 Subject: [PATCH 03/16] quota lookup by ref in decomposedfs --- go.mod | 2 +- go.sum | 4 ++-- pkg/storage/fs/owncloud/owncloud_unix.go | 3 ++- pkg/storage/fs/owncloud/owncloud_windows.go | 3 ++- .../fs/owncloudsql/owncloudsql_unix.go | 3 ++- .../fs/owncloudsql/owncloudsql_windows.go | 3 ++- .../utils/decomposedfs/decomposedfs.go | 10 ++++++-- pkg/storage/utils/decomposedfs/lookup.go | 10 +++++--- pkg/storage/utils/decomposedfs/node/node.go | 24 ++++++++++--------- pkg/storage/utils/decomposedfs/upload.go | 3 ++- 10 files changed, 41 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 032a27b317..3e97f71c06 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30 + github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 diff --git a/go.sum b/go.sum index 9f762f9bbb..2e257cad81 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30 h1:pbropYqSquVy4hWqKEhbVnITo0hJxdfl5Cew8zuiwkI= -github.com/cs3org/go-cs3apis v0.0.0-20211006080433-1c957dea0f30/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11 h1:cc/8fdzWdr/wAZOXb29J8bnXjo1poCMCLwhlFBlvhfI= +github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/storage/fs/owncloud/owncloud_unix.go b/pkg/storage/fs/owncloud/owncloud_unix.go index 0bb88308c8..26eca2d0af 100755 --- a/pkg/storage/fs/owncloud/owncloud_unix.go +++ b/pkg/storage/fs/owncloud/owncloud_unix.go @@ -30,6 +30,7 @@ import ( "strings" "syscall" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ) @@ -68,7 +69,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *ocfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *ocfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloud/owncloud_windows.go b/pkg/storage/fs/owncloud/owncloud_windows.go index 69dbbeddb3..42d743df07 100644 --- a/pkg/storage/fs/owncloud/owncloud_windows.go +++ b/pkg/storage/fs/owncloud/owncloud_windows.go @@ -29,6 +29,7 @@ import ( "os" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "golang.org/x/sys/windows" ) @@ -55,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *ocfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *ocfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloudsql/owncloudsql_unix.go b/pkg/storage/fs/owncloudsql/owncloudsql_unix.go index a63f712df6..d9f002b7b9 100755 --- a/pkg/storage/fs/owncloudsql/owncloudsql_unix.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql_unix.go @@ -30,6 +30,7 @@ import ( "strings" "syscall" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ) @@ -68,7 +69,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return strings.Trim(etag, "\"") } -func (fs *owncloudsqlfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *owncloudsqlfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloudsql/owncloudsql_windows.go b/pkg/storage/fs/owncloudsql/owncloudsql_windows.go index 6abd8e1098..a38f99840e 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql_windows.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql_windows.go @@ -29,6 +29,7 @@ import ( "os" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "golang.org/x/sys/windows" ) @@ -55,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *owncloudsqlfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *owncloudsqlfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 41d8cee8a1..e88f4fe91b 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -130,8 +130,14 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error { // TODO Document in the cs3 should we return quota or free space? func (fs *Decomposedfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (total uint64, inUse uint64, err error) { var n *node.Node - if n, err = fs.lu.HomeOrRootNode(ctx); err != nil { - return 0, 0, err + if req.Ref != nil { + if n, err = fs.lu.NodeFromResource(ctx, req.Ref); err != nil { + return 0, 0, err + } + } else { + if n, err = fs.lu.HomeOrRootNode(ctx); err != nil { + return 0, 0, err + } } if !n.Exists { diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index fb85fc1a6c..a5285217de 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -45,11 +45,12 @@ func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) if ref.ResourceId != nil { // check if a storage space reference is used // currently, the decomposed fs uses the root node id as the space id - n, err := lu.NodeFromID(ctx, ref.ResourceId) + spaceRoot, err := lu.NodeFromID(ctx, ref.ResourceId) if err != nil { return nil, err } + n := spaceRoot p := filepath.Clean(ref.Path) if p != "." { // walk the relative path @@ -59,6 +60,8 @@ func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) if err != nil { return nil, err } + // use reference id as space root for relative references + n.SpaceRoot = spaceRoot return n, nil } @@ -78,11 +81,12 @@ func (lu *Lookup) NodeFromPath(ctx context.Context, fn string, followReferences log := appctx.GetLogger(ctx) log.Debug().Interface("fn", fn).Msg("NodeFromPath()") - n, err := lu.HomeOrRootNode(ctx) + root, err := lu.HomeOrRootNode(ctx) if err != nil { return nil, err } + n := root // TODO collect permissions of the current user on every segment fn = filepath.Clean(fn) if fn != "/" && fn != "." { @@ -94,7 +98,7 @@ func (lu *Lookup) NodeFromPath(ctx context.Context, fn string, followReferences return nil, err } } - + n.SpaceRoot = root return n, nil } diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 3b67fc7af4..a375e79c63 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -65,13 +65,14 @@ const ( // Node represents a node in the tree and provides methods to get a Parent or Child instance type Node struct { - ParentID string - ID string - Name string - Blobsize int64 - BlobID string - owner *userpb.UserId - Exists bool + ParentID string + ID string + Name string + Blobsize int64 + BlobID string + owner *userpb.UserId + Exists bool + SpaceRoot *Node lu PathLookup } @@ -608,12 +609,13 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // quota if _, ok := mdKeysMap[QuotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok { var quotaPath string - if r, err := n.lu.HomeOrRootNode(ctx); err == nil { - quotaPath = r.InternalPath() - readQuotaIntoOpaque(ctx, quotaPath, ri) + if n.SpaceRoot != nil { + quotaPath = n.SpaceRoot.InternalPath() } else { - sublog.Error().Err(err).Msg("error determining home or root node for quota") + sublog.Error().Err(err).Msg("error determining the space root node for quota") } + quotaPath = n.InternalPath() + readQuotaIntoOpaque(ctx, quotaPath, ri) } // only read the requested metadata attributes diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index c810a5d03d..1287fdaebc 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -735,7 +735,8 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo } func checkQuota(ctx context.Context, fs *Decomposedfs, fileSize uint64) (quotaSufficient bool, err error) { - total, inUse, err := fs.GetQuota(ctx) + req := &provider.GetQuotaRequest{} + total, inUse, err := fs.GetQuota(ctx, req) if err != nil { switch err.(type) { case errtypes.NotFound: From 7b573b481eef6fa11acb2c7b10c85a9f11e615e7 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Mon, 11 Oct 2021 14:45:01 +0200 Subject: [PATCH 04/16] quota lookup by ref in decomposedfs --- .../unreleased/get-quota-storage-space.md | 7 ++++++ go.sum | 10 --------- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 ++ .../utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/decomposedfs/lookup.go | 22 +++++++++---------- pkg/storage/utils/decomposedfs/node/node.go | 19 ++++++++++------ pkg/storage/utils/decomposedfs/spaces.go | 8 +++++++ pkg/storage/utils/decomposedfs/tree/tree.go | 8 +++++-- pkg/storage/utils/decomposedfs/upload.go | 6 ++++- 9 files changed, 52 insertions(+), 32 deletions(-) create mode 100644 changelog/unreleased/get-quota-storage-space.md diff --git a/changelog/unreleased/get-quota-storage-space.md b/changelog/unreleased/get-quota-storage-space.md new file mode 100644 index 0000000000..9fd8208dd1 --- /dev/null +++ b/changelog/unreleased/get-quota-storage-space.md @@ -0,0 +1,7 @@ +Enhancement: Add a reference parameter to the getQuota request + +Implementation of [cs3org/cs3apis#147](https://github.com/cs3org/cs3apis/pull/147) + +Make the cs3apis accept a Reference in the getQuota Request to limit the call to a specific storage space. + +https://github.com/cs3org/reva/issues/2152 \ No newline at end of file diff --git a/go.sum b/go.sum index 2e257cad81..4fed162719 100644 --- a/go.sum +++ b/go.sum @@ -486,24 +486,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -<<<<<<< HEAD go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0 h1:Wx7nFnvCaissIUZxPkBqDz2963Z+Cl+PkYbDKzTxDqQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -======= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.24.0 h1:1hCzM7mwQbFQgk3Q4lAVEsGV6NB4Uj6Jt3EU+OiSBc8= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.24.0/go.mod h1:O0cG0vP6TP3c323kh70JmeG1jN69Sn9Z5HxgmeASFWY= -go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= ->>>>>>> update cs3apis go.opentelemetry.io/otel v1.0.1 h1:4XKyXmfqJLOQ7feyV5DB6gsBFZ0ltB8vLtp6pj4JIcc= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel/exporters/jaeger v1.0.1 h1:fg9udWIWWJMAT+Gq2ATFd/DFy3OZvKEZy9VK2amxvkw= go.opentelemetry.io/otel/exporters/jaeger v1.0.1/go.mod h1:85Ym3qknJdIdfRzYS9Ofy9NeLi9gKPFzFDBEHCKpfXI= go.opentelemetry.io/otel/sdk v1.0.1 h1:wXxFEWGo7XfXupPwVJvTBOaPBC9FEg0wB8hMNrKk+cA= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -<<<<<<< HEAD -======= -go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= ->>>>>>> update cs3apis go.opentelemetry.io/otel/trace v1.0.1 h1:StTeIH6Q3G4r0Fiw34LTokUFESZgIDUr0qIJ7mKmAfw= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index 6925a12c46..c3809e354b 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -882,6 +882,8 @@ var _ = Describe("Nextcloud", func() { nc, called, teardown := setUpNextcloudServer() defer teardown() maxBytes, maxFiles, err := nc.GetQuota(ctx) + nc.SetHTTPClient(mock) + maxBytes, maxFiles, err := nc.GetQuota(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(maxBytes).To(Equal(uint64(456))) Expect(maxFiles).To(Equal(uint64(123))) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index e88f4fe91b..c1fffd27a0 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -295,7 +295,7 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) err = fs.tp.CreateDir(ctx, n) - if fs.o.TreeTimeAccounting { + if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { nodePath := n.InternalPath() // mark the home node as the end of propagation if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index a5285217de..94379b3012 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -49,22 +49,22 @@ func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) if err != nil { return nil, err } - n := spaceRoot - p := filepath.Clean(ref.Path) - if p != "." { - // walk the relative path - n, err = lu.WalkPath(ctx, n, p, false, func(ctx context.Context, n *node.Node) error { - return nil - }) - if err != nil { - return nil, err + // is this a relative reference? + if ref.Path != "" { + p := filepath.Clean(ref.Path) + if p != "." { + // walk the relative path + n, err = lu.WalkPath(ctx, n, p, false, func(ctx context.Context, n *node.Node) error { + return nil + }) + if err != nil { + return nil, err + } } // use reference id as space root for relative references n.SpaceRoot = spaceRoot - return n, nil } - return n, nil } diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index a375e79c63..bfac1410d7 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -240,9 +240,10 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { if err != nil { if os.IsNotExist(err) || isNotDir(err) { c := &Node{ - lu: n.lu, - ParentID: n.ID, - Name: name, + lu: n.lu, + ParentID: n.ID, + Name: name, + SpaceRoot: n.SpaceRoot, } return c, nil // if the file does not exist we return a node that has Exists = false } @@ -269,8 +270,9 @@ func (n *Node) Parent() (p *Node, err error) { return nil, fmt.Errorf("Decomposedfs: root has no parent") } p = &Node{ - lu: n.lu, - ID: n.ParentID, + lu: n.lu, + ID: n.ParentID, + SpaceRoot: n.SpaceRoot, } parentPath := n.lu.InternalPath(n.ParentID) @@ -612,9 +614,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi if n.SpaceRoot != nil { quotaPath = n.SpaceRoot.InternalPath() } else { - sublog.Error().Err(err).Msg("error determining the space root node for quota") + root, err := n.lu.HomeOrRootNode(ctx) + if err != nil { + sublog.Error().Err(err).Msg("error determining the space root node for quota") + } + quotaPath = root.InternalPath() } - quotaPath = n.InternalPath() readQuotaIntoOpaque(ctx, quotaPath, ri) } diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 7bc82296c6..473fdabab6 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -66,6 +66,14 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return nil, err } + // always enable propagation on the storage space root + nodePath := n.InternalPath() + // mark the space root node as the end of propagation + if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") + return nil, err + } + if err := fs.createHiddenSpaceFolder(ctx, n); err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index 6d0da24d00..30752f16cb 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -578,8 +578,12 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node) (err error) { // is propagation enabled for the parent node? var root *node.Node - if root, err = t.lookup.HomeOrRootNode(ctx); err != nil { - return + if n.SpaceRoot == nil { + if root, err = t.lookup.HomeOrRootNode(ctx); err != nil { + return + } + } else { + root = n.SpaceRoot } // use a sync time and don't rely on the mtime of the current node, as the stat might not change when a rename happened too quickly diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 1287fdaebc..31a70c2db6 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -252,12 +252,14 @@ func (fs *Decomposedfs) NewUpload(ctx context.Context, info tusd.FileInfo) (uplo } info.Storage = map[string]string{ + // Todo: add storage space root "Type": "OCISStore", "BinPath": binPath, "NodeId": n.ID, "NodeParentId": n.ParentID, "NodeName": n.Name, + "SpaceRoot": n.SpaceRoot.ID, "Idp": usr.Id.Idp, "UserId": usr.Id.OpaqueId, @@ -474,7 +476,9 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { nil, upload.fs.lu, ) - + n.SpaceRoot = &node.Node{ + ID: upload.info.Storage["SpaceRoot"], + } if n.ID == "" { n.ID = uuid.New().String() } From 14b2d8b57665f5369e6f787c2802270c30a7bcdb Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Tue, 12 Oct 2021 15:17:08 +0200 Subject: [PATCH 05/16] unwrap path in getQuota --- .../storageprovider/storageprovider.go | 12 +++++++++- .../utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/decomposedfs/upload.go | 22 ++++++++++++------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 0aa25edc74..ea9b67782c 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -1181,7 +1181,17 @@ func (s *service) CreateSymlink(ctx context.Context, req *provider.CreateSymlink } func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (*provider.GetQuotaResponse, error) { - total, used, err := s.storage.GetQuota(ctx, req) + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + return &provider.GetQuotaResponse{ + Status: status.NewInternal(ctx, err, "error unwrapping path"), + }, nil + } + newReq := &provider.GetQuotaRequest{ + Ref: newRef, + Opaque: req.Opaque, + } + total, used, err := s.storage.GetQuota(ctx, newReq) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index c1fffd27a0..5696f7cd00 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -212,7 +212,7 @@ func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { return } - if fs.o.TreeTimeAccounting { + if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { homePath := h.InternalPath() // mark the home node as the end of propagation if err = xattr.Set(homePath, xattrs.PropagationAttr, []byte("1")); err != nil { diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 31a70c2db6..42be84e4ec 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -157,7 +157,7 @@ func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Refere log.Debug().Interface("info", info).Interface("node", n).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename") - _, err = checkQuota(ctx, fs, uint64(info.Size)) + _, err = checkQuota(ctx, fs, n.SpaceRoot, uint64(info.Size)) if err != nil { return nil, err } @@ -462,11 +462,6 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { return } - _, err = checkQuota(upload.ctx, upload.fs, uint64(fi.Size())) - if err != nil { - return err - } - n := node.New( upload.info.Storage["NodeId"], upload.info.Storage["NodeParentId"], @@ -479,6 +474,11 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { n.SpaceRoot = &node.Node{ ID: upload.info.Storage["SpaceRoot"], } + _, err = checkQuota(upload.ctx, upload.fs, n.SpaceRoot, uint64(fi.Size())) + if err != nil { + return err + } + if n.ID == "" { n.ID = uuid.New().String() } @@ -738,8 +738,14 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo return } -func checkQuota(ctx context.Context, fs *Decomposedfs, fileSize uint64) (quotaSufficient bool, err error) { - req := &provider.GetQuotaRequest{} +func checkQuota(ctx context.Context, fs *Decomposedfs, spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { + req := &provider.GetQuotaRequest{ + Ref: &provider.Reference{ + ResourceId: &provider.ResourceId{ + OpaqueId: spaceRoot.ID, + }, + }, + } total, inUse, err := fs.GetQuota(ctx, req) if err != nil { switch err.(type) { From ab7054c43951d6064c392d8c439a362327f96cb6 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 13 Oct 2021 15:48:43 +0200 Subject: [PATCH 06/16] fix root node --- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 -- pkg/storage/utils/decomposedfs/lookup.go | 4 +++- pkg/storage/utils/decomposedfs/upload_test.go | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index c3809e354b..6a94f73a20 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -881,8 +881,6 @@ var _ = Describe("Nextcloud", func() { It("calls the GetQuota endpoint", func() { nc, called, teardown := setUpNextcloudServer() defer teardown() - maxBytes, maxFiles, err := nc.GetQuota(ctx) - nc.SetHTTPClient(mock) maxBytes, maxFiles, err := nc.GetQuota(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(maxBytes).To(Equal(uint64(456))) diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 94379b3012..f8dcb4aaa5 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -133,7 +133,9 @@ func (lu *Lookup) Path(ctx context.Context, n *node.Node) (p string, err error) // RootNode returns the root node of the storage func (lu *Lookup) RootNode(ctx context.Context) (*node.Node, error) { - return node.New("root", "", "", 0, "", nil, lu), nil + n := node.New("root", "", "", 0, "", nil, lu) + n.Exists = true + return n, nil } // HomeNode returns the home node of a user diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go index 01ef9cb4b3..4804fddf6e 100644 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ b/pkg/storage/utils/decomposedfs/upload_test.go @@ -96,6 +96,7 @@ var _ = Describe("File uploads", func() { Context("with insufficient permissions", func() { BeforeEach(func() { permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) + permissions.On("AssemblePermissions", mock.Anything, mock.Anything).Return(provider.ResourcePermissions{GetQuota: true}, nil) }) Describe("InitiateUpload", func() { @@ -109,6 +110,7 @@ var _ = Describe("File uploads", func() { Context("with sufficient permissions", func() { BeforeEach(func() { permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + permissions.On("AssemblePermissions", mock.Anything, mock.Anything).Return(provider.ResourcePermissions{GetQuota: true}, nil) }) Describe("InitiateUpload", func() { From ea14bfa51144ebba632340f8abfe018ba607851b Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 13 Oct 2021 16:51:07 +0200 Subject: [PATCH 07/16] Fix wrong space root during upload --- pkg/storage/utils/decomposedfs/node/node.go | 17 ++++++++++------- pkg/storage/utils/decomposedfs/upload.go | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index bfac1410d7..bb6497ab2a 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -611,16 +611,19 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // quota if _, ok := mdKeysMap[QuotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok { var quotaPath string - if n.SpaceRoot != nil { - quotaPath = n.SpaceRoot.InternalPath() - } else { + if n.SpaceRoot == nil { root, err := n.lu.HomeOrRootNode(ctx) - if err != nil { - sublog.Error().Err(err).Msg("error determining the space root node for quota") + if err == nil { + quotaPath = root.InternalPath() + } else { + sublog.Debug().Err(err).Msg("error determining the space root node for quota") } - quotaPath = root.InternalPath() + } else { + quotaPath = n.SpaceRoot.InternalPath() + } + if quotaPath != "" { + readQuotaIntoOpaque(ctx, quotaPath, ri) } - readQuotaIntoOpaque(ctx, quotaPath, ri) } // only read the requested metadata attributes diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 42be84e4ec..e8011532f6 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -132,6 +132,9 @@ func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Refere "dir": filepath.Dir(relative), }, Size: uploadLength, + Storage: map[string]string{ + "SpaceRoot": n.SpaceRoot.ID, + }, } if metadata != nil { @@ -250,16 +253,23 @@ func (fs *Decomposedfs) NewUpload(ctx context.Context, info tusd.FileInfo) (uplo if err != nil { return nil, errors.Wrap(err, "Decomposedfs: error determining owner") } + var spaceRoot string + if info.Storage != nil { + if spaceRoot, ok = info.Storage["SpaceRoot"]; !ok { + spaceRoot = n.SpaceRoot.ID + } + } else { + spaceRoot = n.SpaceRoot.ID + } info.Storage = map[string]string{ - // Todo: add storage space root "Type": "OCISStore", "BinPath": binPath, "NodeId": n.ID, "NodeParentId": n.ParentID, "NodeName": n.Name, - "SpaceRoot": n.SpaceRoot.ID, + "SpaceRoot": spaceRoot, "Idp": usr.Id.Idp, "UserId": usr.Id.OpaqueId, @@ -739,14 +749,13 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo } func checkQuota(ctx context.Context, fs *Decomposedfs, spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { - req := &provider.GetQuotaRequest{ + total, inUse, err := fs.GetQuota(ctx, &provider.GetQuotaRequest{ Ref: &provider.Reference{ ResourceId: &provider.ResourceId{ OpaqueId: spaceRoot.ID, }, }, - } - total, inUse, err := fs.GetQuota(ctx, req) + }) if err != nil { switch err.(type) { case errtypes.NotFound: From 9bf7ffb32b490307c409297b5349c92c3beeb926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 14 Oct 2021 10:01:44 +0000 Subject: [PATCH 08/16] remember space root when walking the path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/decomposedfs/lookup.go | 3 +++ pkg/storage/utils/decomposedfs/node/node.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index f8dcb4aaa5..49f67d863e 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -92,6 +92,9 @@ func (lu *Lookup) NodeFromPath(ctx context.Context, fn string, followReferences if fn != "/" && fn != "." { n, err = lu.WalkPath(ctx, n, fn, followReferences, func(ctx context.Context, n *node.Node) error { log.Debug().Interface("node", n).Msg("NodeFromPath() walk") + if n.SpaceRoot != root { + root = n.SpaceRoot + } return nil }) if err != nil { diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index bb6497ab2a..84f05fcfa7 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -192,6 +192,10 @@ func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error default: return nil, errtypes.InternalError(err.Error()) } + // check if this is a space root + if _, err = xattr.Get(nodePath, xattrs.SpaceNameAttr); err == nil { + n.SpaceRoot = n + } // lookup name in extended attributes if attrBytes, err = xattr.Get(nodePath, xattrs.NameAttr); err == nil { n.Name = string(attrBytes) From 838f2408654430ceefa4d9a3fdeee18914db0013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 14 Oct 2021 10:47:50 +0000 Subject: [PATCH 09/16] set space name for personal space, fix space root identification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/decomposedfs/decomposedfs.go | 4 ++++ pkg/storage/utils/decomposedfs/lookup.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 5696f7cd00..4e9de4f8d4 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -221,6 +221,10 @@ func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { } } + if err := n.SetMetadata(xattrs.SpaceNameAttr, u.DisplayName); err != nil { + return err + } + // add storage space if err := fs.createStorageSpace(ctx, "personal", h.ID); err != nil { return err diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 49f67d863e..974edd0e4c 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -92,7 +92,7 @@ func (lu *Lookup) NodeFromPath(ctx context.Context, fn string, followReferences if fn != "/" && fn != "." { n, err = lu.WalkPath(ctx, n, fn, followReferences, func(ctx context.Context, n *node.Node) error { log.Debug().Interface("node", n).Msg("NodeFromPath() walk") - if n.SpaceRoot != root { + if n.SpaceRoot != nil && n.SpaceRoot != root { root = n.SpaceRoot } return nil From 78bf48ddeecacf1c91210a5266bad100826c9ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 14 Oct 2021 10:48:30 +0000 Subject: [PATCH 10/16] do not use higher level GetQuota to omit permission check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/decomposedfs/upload.go | 43 ++++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index e8011532f6..04c6c6cc73 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -31,6 +31,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "time" @@ -749,23 +750,37 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo } func checkQuota(ctx context.Context, fs *Decomposedfs, spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { - total, inUse, err := fs.GetQuota(ctx, &provider.GetQuotaRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - OpaqueId: spaceRoot.ID, - }, - }, - }) + + ri, err := spaceRoot.AsResourceInfo(ctx, nil, []string{"treesize", "quota"}, true) if err != nil { - switch err.(type) { - case errtypes.NotFound: - // no quota for this storage (eg. no user context) - return true, nil - default: - return false, err + return false, err + } + + quotaStr := node.QuotaUnknown + if ri.Opaque != nil && ri.Opaque.Map != nil && ri.Opaque.Map["quota"] != nil && ri.Opaque.Map["quota"].Decoder == "plain" { + quotaStr = string(ri.Opaque.Map["quota"].Value) + } + + avail, err := fs.getAvailableSize(spaceRoot.InternalPath()) + if err != nil { + return false, err + } + total := avail + ri.Size + + switch { + case quotaStr == node.QuotaUncalculated, quotaStr == node.QuotaUnknown, quotaStr == node.QuotaUnlimited: + // best we can do is return current total + // TODO indicate unlimited total? -> in opaque data? + // TODO distinguish two errors: out of quota vs oud of disk space + // - the user might not be out of quota, but the disk might still be full -> admin needs to take care, actually LOG an ERR, ui needs to explain + default: + if quota, err := strconv.ParseUint(quotaStr, 10, 64); err == nil { + if total > quota { + total = quota + } } } - if !(total == 0) && fileSize > total-inUse { + if !(total == 0) && fileSize > total-ri.Size { return false, errtypes.InsufficientStorage("quota exceeded") } return true, nil From 0edd79fd6531f10744b8a590f36d674a4e34aa32 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Thu, 14 Oct 2021 13:12:17 +0200 Subject: [PATCH 11/16] add lookup when creating the space root --- pkg/storage/utils/decomposedfs/upload.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 04c6c6cc73..85c081794f 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -482,9 +482,8 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { nil, upload.fs.lu, ) - n.SpaceRoot = &node.Node{ - ID: upload.info.Storage["SpaceRoot"], - } + n.SpaceRoot = node.New(upload.info.Storage["SpaceRoot"], "", "", 0, "", nil, upload.fs.lu) + _, err = checkQuota(upload.ctx, upload.fs, n.SpaceRoot, uint64(fi.Size())) if err != nil { return err From 2ef147d5b22276f9fcabb89b3fe8a127f254c415 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Thu, 14 Oct 2021 15:06:04 +0200 Subject: [PATCH 12/16] enforce quota on spaces --- pkg/storage/utils/decomposedfs/upload.go | 40 ++++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 85c081794f..0a20360710 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -43,9 +43,11 @@ import ( "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" + "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/utils" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/pkg/xattr" "github.com/rs/zerolog" tusd "github.com/tus/tusd/pkg/handler" ) @@ -749,38 +751,36 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo } func checkQuota(ctx context.Context, fs *Decomposedfs, spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { - - ri, err := spaceRoot.AsResourceInfo(ctx, nil, []string{"treesize", "quota"}, true) + used, err := spaceRoot.GetTreeSize() if err != nil { return false, err } - quotaStr := node.QuotaUnknown - if ri.Opaque != nil && ri.Opaque.Map != nil && ri.Opaque.Map["quota"] != nil && ri.Opaque.Map["quota"].Decoder == "plain" { - quotaStr = string(ri.Opaque.Map["quota"].Value) + quotaB, err := xattr.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) + if err != nil { + return false, err } - avail, err := fs.getAvailableSize(spaceRoot.InternalPath()) + total, err := strconv.ParseUint(string(quotaB), 10, 64) if err != nil { return false, err } - total := avail + ri.Size - switch { - case quotaStr == node.QuotaUncalculated, quotaStr == node.QuotaUnknown, quotaStr == node.QuotaUnlimited: - // best we can do is return current total - // TODO indicate unlimited total? -> in opaque data? - // TODO distinguish two errors: out of quota vs oud of disk space - // - the user might not be out of quota, but the disk might still be full -> admin needs to take care, actually LOG an ERR, ui needs to explain - default: - if quota, err := strconv.ParseUint(quotaStr, 10, 64); err == nil { - if total > quota { - total = quota - } - } + if !enoughFreeSpace(fs, spaceRoot.InternalPath(), fileSize) { + return false, errtypes.InsufficientStorage("disk full") } - if !(total == 0) && fileSize > total-ri.Size { + + if fileSize > total-used || total < used { return false, errtypes.InsufficientStorage("quota exceeded") } return true, nil } + +func enoughFreeSpace(fs *Decomposedfs, path string, fileSize uint64) bool { + avalB, err := fs.getAvailableSize(path) + if err != nil { + return false + } + + return avalB > fileSize +} From 3598af2776afb7e80c38d80f255aaeaecb219de0 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Thu, 14 Oct 2021 15:14:25 +0200 Subject: [PATCH 13/16] remove unnecessary mock --- pkg/storage/utils/decomposedfs/upload_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go index 4804fddf6e..01ef9cb4b3 100644 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ b/pkg/storage/utils/decomposedfs/upload_test.go @@ -96,7 +96,6 @@ var _ = Describe("File uploads", func() { Context("with insufficient permissions", func() { BeforeEach(func() { permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - permissions.On("AssemblePermissions", mock.Anything, mock.Anything).Return(provider.ResourcePermissions{GetQuota: true}, nil) }) Describe("InitiateUpload", func() { @@ -110,7 +109,6 @@ var _ = Describe("File uploads", func() { Context("with sufficient permissions", func() { BeforeEach(func() { permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - permissions.On("AssemblePermissions", mock.Anything, mock.Anything).Return(provider.ResourcePermissions{GetQuota: true}, nil) }) Describe("InitiateUpload", func() { From fbf11acfef6fb317ec71c804c4d056d856efef6c Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Thu, 14 Oct 2021 15:28:46 +0200 Subject: [PATCH 14/16] guard against quota calculation --- pkg/storage/utils/decomposedfs/upload.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 0a20360710..5e69270d26 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -751,32 +751,22 @@ func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Uplo } func checkQuota(ctx context.Context, fs *Decomposedfs, spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { - used, err := spaceRoot.GetTreeSize() - if err != nil { - return false, err - } - - quotaB, err := xattr.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) - if err != nil { - return false, err - } - - total, err := strconv.ParseUint(string(quotaB), 10, 64) - if err != nil { - return false, err - } + used, _ := spaceRoot.GetTreeSize() + quotaB, _ := xattr.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) + total, _ := strconv.ParseUint(string(quotaB), 10, 64) - if !enoughFreeSpace(fs, spaceRoot.InternalPath(), fileSize) { + enoughDiskSpace := enoughDiskSpace(fs, spaceRoot.InternalPath(), fileSize) + if !enoughDiskSpace { return false, errtypes.InsufficientStorage("disk full") } - if fileSize > total-used || total < used { + if (fileSize > total-used || total < used) && !enoughDiskSpace { return false, errtypes.InsufficientStorage("quota exceeded") } return true, nil } -func enoughFreeSpace(fs *Decomposedfs, path string, fileSize uint64) bool { +func enoughDiskSpace(fs *Decomposedfs, path string, fileSize uint64) bool { avalB, err := fs.getAvailableSize(path) if err != nil { return false From c5ca8080f5876333335a2c07dfac8fd6b7e70ede Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Thu, 14 Oct 2021 15:48:34 +0200 Subject: [PATCH 15/16] fix windows build --- pkg/storage/utils/localfs/localfs_windows.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/storage/utils/localfs/localfs_windows.go b/pkg/storage/utils/localfs/localfs_windows.go index 1cc33dd6c9..2456f3298f 100644 --- a/pkg/storage/utils/localfs/localfs_windows.go +++ b/pkg/storage/utils/localfs/localfs_windows.go @@ -29,6 +29,7 @@ import ( "os" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "golang.org/x/sys/windows" ) @@ -55,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *localfs) GetQuota(ctx context.Context) (uint64, uint64, error) { +func (fs *localfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root From 3245082a941a88d7d4c436d9de27c3631c7fb6a7 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Thu, 14 Oct 2021 18:16:05 +0200 Subject: [PATCH 16/16] use reference in the storage interface --- internal/grpc/services/storageprovider/storageprovider.go | 6 +----- pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/owncloud/owncloud_unix.go | 2 +- pkg/storage/fs/owncloud/owncloud_windows.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql_unix.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql_windows.go | 2 +- pkg/storage/fs/s3/s3.go | 2 +- pkg/storage/storage.go | 2 +- pkg/storage/utils/decomposedfs/decomposedfs.go | 6 +++--- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/localfs/localfs_unix.go | 2 +- pkg/storage/utils/localfs/localfs_windows.go | 2 +- 12 files changed, 14 insertions(+), 18 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index ea9b67782c..19e363c9ef 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -1187,11 +1187,7 @@ func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) ( Status: status.NewInternal(ctx, err, "error unwrapping path"), }, nil } - newReq := &provider.GetQuotaRequest{ - Ref: newRef, - Opaque: req.Opaque, - } - total, used, err := s.storage.GetQuota(ctx, newReq) + total, used, err := s.storage.GetQuota(ctx, newRef) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 10d6af265b..aefe5e26d7 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -679,7 +679,7 @@ func (nc *StorageDriver) ListGrants(ctx context.Context, ref *provider.Reference } // GetQuota as defined in the storage.FS interface -func (nc *StorageDriver) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (nc *StorageDriver) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { log := appctx.GetLogger(ctx) log.Info().Msg("GetQuota") diff --git a/pkg/storage/fs/owncloud/owncloud_unix.go b/pkg/storage/fs/owncloud/owncloud_unix.go index 26eca2d0af..61cc65433d 100755 --- a/pkg/storage/fs/owncloud/owncloud_unix.go +++ b/pkg/storage/fs/owncloud/owncloud_unix.go @@ -69,7 +69,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *ocfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *ocfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloud/owncloud_windows.go b/pkg/storage/fs/owncloud/owncloud_windows.go index 42d743df07..80e86e4cd4 100644 --- a/pkg/storage/fs/owncloud/owncloud_windows.go +++ b/pkg/storage/fs/owncloud/owncloud_windows.go @@ -56,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *ocfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *ocfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloudsql/owncloudsql_unix.go b/pkg/storage/fs/owncloudsql/owncloudsql_unix.go index d9f002b7b9..95ef01cab2 100755 --- a/pkg/storage/fs/owncloudsql/owncloudsql_unix.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql_unix.go @@ -69,7 +69,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return strings.Trim(etag, "\"") } -func (fs *owncloudsqlfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *owncloudsqlfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/owncloudsql/owncloudsql_windows.go b/pkg/storage/fs/owncloudsql/owncloudsql_windows.go index a38f99840e..679c9e77d0 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql_windows.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql_windows.go @@ -56,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *owncloudsqlfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *owncloudsqlfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 7091d8514e..7636458a8b 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -268,7 +268,7 @@ func (fs *s3FS) UpdateGrant(ctx context.Context, ref *provider.Reference, g *pro return errtypes.NotSupported("s3: operation not supported") } -func (fs *s3FS) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *s3FS) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { return 0, 0, nil } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4179b4af02..31a021e387 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -52,7 +52,7 @@ type FS interface { RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) - GetQuota(ctx context.Context, req *provider.GetQuotaRequest) ( /*TotalBytes*/ uint64 /*UsedBytes*/, uint64, error) + GetQuota(ctx context.Context, ref *provider.Reference) ( /*TotalBytes*/ uint64 /*UsedBytes*/, uint64, error) CreateReference(ctx context.Context, path string, targetURI *url.URL) error Shutdown(ctx context.Context) error SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 4e9de4f8d4..59c6b7ecb4 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -128,10 +128,10 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error { // GetQuota returns the quota available // TODO Document in the cs3 should we return quota or free space? -func (fs *Decomposedfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (total uint64, inUse uint64, err error) { +func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, err error) { var n *node.Node - if req.Ref != nil { - if n, err = fs.lu.NodeFromResource(ctx, req.Ref); err != nil { + if ref != nil { + if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { return 0, 0, err } } else { diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 4f522ad340..e25eeebd11 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -973,7 +973,7 @@ func (fs *eosfs) CreateStorageSpace(ctx context.Context, req *provider.CreateSto return nil, fmt.Errorf("unimplemented: CreateStorageSpace") } -func (fs *eosfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *eosfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { u, err := getUser(ctx) if err != nil { return 0, 0, errors.Wrap(err, "eosfs: no user in ctx") diff --git a/pkg/storage/utils/localfs/localfs_unix.go b/pkg/storage/utils/localfs/localfs_unix.go index 6f9b8aa61c..61cdaf9e86 100644 --- a/pkg/storage/utils/localfs/localfs_unix.go +++ b/pkg/storage/utils/localfs/localfs_unix.go @@ -67,7 +67,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *localfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *localfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root diff --git a/pkg/storage/utils/localfs/localfs_windows.go b/pkg/storage/utils/localfs/localfs_windows.go index 2456f3298f..235f1867d0 100644 --- a/pkg/storage/utils/localfs/localfs_windows.go +++ b/pkg/storage/utils/localfs/localfs_windows.go @@ -56,7 +56,7 @@ func calcEtag(ctx context.Context, fi os.FileInfo) string { return fmt.Sprintf("\"%s\"", strings.Trim(etag, "\"")) } -func (fs *localfs) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (uint64, uint64, error) { +func (fs *localfs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { // TODO quota of which storage space? // we could use the logged in user, but when a user has access to multiple storages this falls short // for now return quota of root