From 44619121ae9acff4b2e686292fe154790cabbf1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 27 Nov 2023 09:15:13 +0100 Subject: [PATCH 1/5] Implement TUS uploads for ocm --- .../ocmshareprovider/ocmshareprovider.go | 10 +- internal/http/services/owncloud/ocdav/put.go | 1 + pkg/ocm/storage/received/ocm.go | 73 ++-- pkg/ocm/storage/received/upload.go | 394 ++++++++++++++++++ 4 files changed, 434 insertions(+), 44 deletions(-) create mode 100644 pkg/ocm/storage/received/upload.go diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 88d775eafe..0f68136fdf 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -515,6 +515,14 @@ func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateRec func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMShareRequest) (*ocm.GetReceivedOCMShareResponse, error) { user := ctxpkg.ContextMustGetUser(ctx) + if user.Id.GetType() == userpb.UserType_USER_TYPE_SERVICE { + var uid userpb.UserId + _ = utils.ReadJSONFromOpaque(req.Opaque, "userid", &uid) + user = &userpb.User{ + Id: &uid, + } + } + ocmshare, err := s.repo.GetReceivedShare(ctx, user, req.Ref) if err != nil { if errors.Is(err, share.ErrShareNotFound) { @@ -523,7 +531,7 @@ func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedO }, nil } return &ocm.GetReceivedOCMShareResponse{ - Status: status.NewInternal(ctx, "error getting received share"), + Status: status.NewInternal(ctx, "error getting received share: "+err.Error()), }, nil } diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index e648582c0d..cb52ea695a 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -137,6 +137,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ length, err := getContentLength(w, r) if err != nil { + log.Error().Err(err).Msg("error getting the content length") w.WriteHeader(http.StatusBadRequest) return } diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go index 95ab95492e..28acb07a0a 100644 --- a/pkg/ocm/storage/received/ocm.go +++ b/pkg/ocm/storage/received/ocm.go @@ -30,6 +30,7 @@ import ( "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" @@ -59,8 +60,11 @@ type driver struct { } type config struct { - GatewaySVC string `mapstructure:"gatewaysvc"` - Insecure bool `mapstructure:"insecure"` + GatewaySVC string `mapstructure:"gatewaysvc"` + Insecure bool `mapstructure:"insecure"` + StorageRoot string `mapstructure:"storage_root"` + ServiceAccountID string `mapstructure:"service_account_id"` + ServiceAccountSecret string `mapstructure:"service_account_secret"` } func (c *config) ApplyDefaults() { @@ -136,15 +140,19 @@ func shareInfoFromReference(ref *provider.Reference) (*ocmpb.ShareId, string) { } -func (d *driver) getWebDAVFromShare(ctx context.Context, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) { +func (d *driver) getWebDAVFromShare(ctx context.Context, forUser *userpb.UserId, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) { // TODO: we may want to cache the share - res, err := d.gateway.GetReceivedOCMShare(ctx, &ocmpb.GetReceivedOCMShareRequest{ + req := &ocmpb.GetReceivedOCMShareRequest{ Ref: &ocmpb.ShareReference{ Spec: &ocmpb.ShareReference_Id{ Id: shareID, }, }, - }) + } + if forUser != nil { + req.Opaque = utils.AppendJSONToOpaque(nil, "userid", forUser) + } + res, err := d.gateway.GetReceivedOCMShare(ctx, req) if err != nil { return nil, "", "", err } @@ -173,10 +181,10 @@ func getWebDAVProtocol(protocols []*ocmpb.Protocol) (*ocmpb.WebDAVProtocol, bool return nil, false } -func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) { +func (d *driver) webdavClient(ctx context.Context, forUser *userpb.UserId, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) { id, rel := shareInfoFromReference(ref) - share, endpoint, secret, err := d.getWebDAVFromShare(ctx, id) + share, endpoint, secret, err := d.getWebDAVFromShare(ctx, forUser, id) if err != nil { return nil, nil, "", err } @@ -199,7 +207,7 @@ func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*go } func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -207,7 +215,7 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { } func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -215,7 +223,7 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { } func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -223,7 +231,7 @@ func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markpro } func (d *driver) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { - client, _, relOld, err := d.webdavClient(ctx, oldRef) + client, _, relOld, err := d.webdavClient(ctx, nil, oldRef) if err != nil { return err } @@ -263,7 +271,7 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc } webdavProtocol, _ := getWebDAVProtocol(share.Protocols) - return &provider.ResourceInfo{ + ri := provider.ResourceInfo{ Type: t, Id: id, MimeType: mime.Detect(f.IsDir(), f.Name()), @@ -278,11 +286,17 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc Checksum: &provider.ResourceChecksum{ Type: provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, }, - }, nil + } + + if f.(gowebdav.File).StatusCode() == 425 { + ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "status", "processing") + } + + return &ri, nil } func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, _ []string) (*provider.ResourceInfo, error) { - client, share, rel, err := d.webdavClient(ctx, ref) + client, share, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } @@ -299,7 +313,7 @@ func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, } func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []string, _ []string) ([]*provider.ResourceInfo, error) { - client, share, rel, err := d.webdavClient(ctx, ref) + client, share, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } @@ -320,35 +334,8 @@ func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []st return res, nil } -func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, _ int64, _ map[string]string) (map[string]string, error) { - shareID, rel := shareInfoFromReference(ref) - p := getPathFromShareIDAndRelPath(shareID, rel) - - return map[string]string{ - "simple": p, - }, nil -} - -func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) { - client, _, rel, err := d.webdavClient(ctx, req.Ref) - if err != nil { - return provider.ResourceInfo{}, err - } - client.SetInterceptor(func(method string, rq *http.Request) { - // Set the content length on the request struct directly instead of the header. - // The content-length header gets reset by the golang http library before - // sendind out the request, resulting in chunked encoding to be used which - // breaks the quota checks in ocdav. - if method == "PUT" { - rq.ContentLength = req.Length - } - }) - - return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0) -} - func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } diff --git a/pkg/ocm/storage/received/upload.go b/pkg/ocm/storage/received/upload.go new file mode 100644 index 0000000000..6b06090ee2 --- /dev/null +++ b/pkg/ocm/storage/received/upload.go @@ -0,0 +1,394 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocm + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "hash/adler32" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/google/uuid" + tusd "github.com/tus/tusd/pkg/handler" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/storage" + "github.com/cs3org/reva/v2/pkg/utils" +) + +var defaultFilePerm = os.FileMode(0664) +var _idRegexp = regexp.MustCompile(".*/([^/]+).info") + +func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + return []storage.UploadSession{}, nil +} +func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + shareID, rel := shareInfoFromReference(ref) + p := getPathFromShareIDAndRelPath(shareID, rel) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := d.NewUpload(ctx, info) + if err != nil { + return nil, err + } + + info, _ = upload.GetInfo(ctx) + + return map[string]string{ + "simple": info.ID, + "tus": info.ID, + }, nil +} + +func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) { + client, _, rel, err := d.webdavClient(ctx, nil, req.Ref) + if err != nil { + return provider.ResourceInfo{}, err + } + client.SetInterceptor(func(method string, rq *http.Request) { + // Set the content length on the request struct directly instead of the header. + // The content-length header gets reset by the golang http library before + // sendind out the request, resulting in chunked encoding to be used which + // breaks the quota checks in ocdav. + if method == "PUT" { + rq.ContentLength = req.Length + } + }) + + return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0) +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (d *driver) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(d) + composer.UseTerminater(d) + composer.UseConcater(d) + composer.UseLengthDeferrer(d) +} + +// AsTerminatableUpload returns a TerminatableUpload +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// the storage needs to implement AsTerminatableUpload +func (d *driver) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*upload) +} + +// AsLengthDeclarableUpload returns a LengthDeclarableUpload +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// the storage needs to implement AsLengthDeclarableUpload +func (d *driver) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*upload) +} + +// AsConcatableUpload returns a ConcatableUpload +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// the storage needs to implement AsConcatableUpload +func (d *driver) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*upload) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +// NewUpload returns a new tus Upload instance +func (d *driver) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) { + return NewUpload(ctx, d, d.c.StorageRoot, info) +} + +// GetUpload returns the Upload for the given upload id +func (d *driver) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + return GetUpload(ctx, d, d.c.StorageRoot, id) +} +func NewUpload(ctx context.Context, d *driver, storageRoot string, info tusd.FileInfo) (tusd.Upload, error) { + if info.MetaData["filename"] == "" { + return nil, errors.New("Decomposedfs: missing filename in metadata") + } + if info.MetaData["dir"] == "" { + return nil, errors.New("Decomposedfs: missing dir in metadata") + } + + uploadRoot := filepath.Join(storageRoot, "uploads") + info.ID = uuid.New().String() + + user, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + return nil, errors.New("no user in context") + } + info.MetaData["user"] = user.GetId().GetOpaqueId() + info.MetaData["idp"] = user.GetId().GetIdp() + + info.Storage = map[string]string{ + "Type": "OCM", + "Path": uploadRoot, + } + + u := &upload{ + Info: info, + Ctx: ctx, + d: d, + } + + err := os.MkdirAll(uploadRoot, 0755) + if err != nil { + return nil, err + } + + file, err := os.OpenFile(u.BinPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + err = u.Persist() + if err != nil { + return nil, err + } + return u, nil +} + +func GetUpload(ctx context.Context, d *driver, storageRoot string, id string) (tusd.Upload, error) { + info := tusd.FileInfo{} + data, err := os.ReadFile(filepath.Join(storageRoot, "uploads", id+".info")) + if err != nil { + return nil, err + } + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + upload := &upload{ + Info: info, + Ctx: ctx, + d: d, + } + return upload, nil +} + +type upload struct { + Info tusd.FileInfo + Ctx context.Context + + d *driver +} + +func (u *upload) InfoPath() string { + return filepath.Join(u.Info.Storage["Path"], u.Info.ID+".info") +} + +func (u *upload) BinPath() string { + return filepath.Join(u.Info.Storage["Path"], u.Info.ID) +} + +func (u *upload) Persist() error { + data, err := json.Marshal(u.Info) + if err != nil { + return err + } + return os.WriteFile(u.InfoPath(), data, defaultFilePerm) +} + +func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum + // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... + // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used + // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for the ocis driver it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil && err != io.ErrUnexpectedEOF { + return n, err + } + + u.Info.Offset += n + return n, u.Persist() +} + +func (u *upload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return u.Info, nil +} + +func (u *upload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(u.BinPath()) +} + +func (u *upload) FinishUpload(ctx context.Context) error { + log := appctx.GetLogger(u.Ctx) + + // calculate the checksum of the written bytes + // they will all be written to the metadata later, so we cannot omit any of them + // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present + // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... + sha1h := sha1.New() + md5h := md5.New() + adler32h := adler32.New() + { + f, err := os.Open(u.BinPath()) + if err != nil { + // we can continue if no oc checksum header is set + log.Info().Err(err).Str("binPath", u.BinPath()).Msg("error opening binPath") + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + + _, err = io.Copy(adler32h, r2) + if err != nil { + log.Info().Err(err).Msg("error copying checksums") + } + } + + // compare if they match the sent checksum + // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads + if u.Info.MetaData["checksum"] != "" { + var err error + parts := strings.SplitN(u.Info.MetaData["checksum"], " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1": + err = u.checkHash(parts[1], sha1h) + case "md5": + err = u.checkHash(parts[1], md5h) + case "adler32": + err = u.checkHash(parts[1], adler32h) + default: + err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if err != nil { + u.cleanup() + return err + } + } + + // send to the remote storage via webdav + // shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"]) + // p := getPathFromShareIDAndRelPath(shareID, rel) + + ctx, err := utils.GetServiceUserContext(u.d.c.ServiceAccountID, u.d.gateway, u.d.c.ServiceAccountSecret) + if err != nil { + return err + } + client, _, rel, err := u.d.webdavClient(ctx, &userpb.UserId{ + OpaqueId: u.Info.MetaData["user"], + Idp: u.Info.MetaData["idp"], + }, &provider.Reference{ + Path: filepath.Join(u.Info.MetaData["dir"], u.Info.MetaData["filename"]), + }) + if err != nil { + u.cleanup() + return err + } + + client.SetInterceptor(func(method string, rq *http.Request) { + // Set the content length on the request struct directly instead of the header. + // The content-length header gets reset by the golang http library before + // sendind out the request, resulting in chunked encoding to be used which + // breaks the quota checks in ocdav. + if method == "PUT" { + rq.ContentLength = u.Info.Size + } + }) + + f, err := os.Open(u.BinPath()) + if err != nil { + return err + } + defer f.Close() + return client.WriteStream(rel, f, 0) +} + +func (u *upload) cleanup() { + _ = os.Remove(u.BinPath()) + _ = os.Remove(u.InfoPath()) +} + +func (u *upload) Terminate(ctx context.Context) error { + u.cleanup() + return nil +} + +func (u *upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) error { + file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*upload) + + src, err := os.Open(fileUpload.BinPath()) + if err != nil { + return err + } + defer src.Close() + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + return nil +} + +func (u *upload) DeclareLength(ctx context.Context, length int64) error { + u.Info.Size = length + u.Info.SizeIsDeferred = false + return nil +} + +func (u *upload) checkHash(expected string, h hash.Hash) error { + if expected != hex.EncodeToString(h.Sum(nil)) { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", u.Info.MetaData["checksum"], h.Sum(nil))) + } + return nil +} From 86d740e632878ce3b5c1fed4321ca1334355522a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 15 Dec 2023 08:42:41 +0100 Subject: [PATCH 2/5] Bump gowebdav to pull in the 425 support --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c15a65a4ed..1bfee2cd9e 100644 --- a/go.mod +++ b/go.mod @@ -231,4 +231,4 @@ require ( replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 -replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 +replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215102054-212d4a4374f6 diff --git a/go.sum b/go.sum index 37514136c0..eb7a21cfc0 100644 --- a/go.sum +++ b/go.sum @@ -425,8 +425,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 h1:m63hhLqbqmLGGPtyTtjTdxae61d9tMbRdKvMaDHWcDs= -github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= +github.com/aduffeck/gowebdav v0.0.0-20231215102054-212d4a4374f6 h1:ws0yvsikTQdmheKINP16tBzAHdttrHwbz/q3Fgl9X1Y= +github.com/aduffeck/gowebdav v0.0.0-20231215102054-212d4a4374f6/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= From 263c2eb6bd4a66ca89202913188876fcf06745d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 15 Dec 2023 11:26:36 +0100 Subject: [PATCH 3/5] Add changelog --- changelog/unreleased/ocm.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/unreleased/ocm.md b/changelog/unreleased/ocm.md index f5e160f388..8e7d77e25b 100644 --- a/changelog/unreleased/ocm.md +++ b/changelog/unreleased/ocm.md @@ -2,4 +2,5 @@ Bugfix: Improve OCM support We fixed several bugs with OCM support. +https://github.com/cs3org/reva/pull/4410 https://github.com/cs3org/reva/pull/4333 From 2c1eacd7360513cb7c275a0615977009d920572c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 15 Dec 2023 11:31:21 +0100 Subject: [PATCH 4/5] Fix linter issues --- pkg/ocm/storage/received/upload.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/ocm/storage/received/upload.go b/pkg/ocm/storage/received/upload.go index 6b06090ee2..6fc6af818f 100644 --- a/pkg/ocm/storage/received/upload.go +++ b/pkg/ocm/storage/received/upload.go @@ -32,7 +32,6 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" "github.com/google/uuid" @@ -48,7 +47,6 @@ import ( ) var defaultFilePerm = os.FileMode(0664) -var _idRegexp = regexp.MustCompile(".*/([^/]+).info") func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { return []storage.UploadSession{}, nil @@ -314,11 +312,11 @@ func (u *upload) FinishUpload(ctx context.Context) error { // shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"]) // p := getPathFromShareIDAndRelPath(shareID, rel) - ctx, err := utils.GetServiceUserContext(u.d.c.ServiceAccountID, u.d.gateway, u.d.c.ServiceAccountSecret) + serviceUserCtx, err := utils.GetServiceUserContext(u.d.c.ServiceAccountID, u.d.gateway, u.d.c.ServiceAccountSecret) if err != nil { return err } - client, _, rel, err := u.d.webdavClient(ctx, &userpb.UserId{ + client, _, rel, err := u.d.webdavClient(serviceUserCtx, &userpb.UserId{ OpaqueId: u.Info.MetaData["user"], Idp: u.Info.MetaData["idp"], }, &provider.Reference{ From 696ffa0ffcc848eed19b03a7bbe4abef1af63ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 15 Dec 2023 15:18:01 +0100 Subject: [PATCH 5/5] Fix simple uploads --- pkg/ocm/storage/received/upload.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/ocm/storage/received/upload.go b/pkg/ocm/storage/received/upload.go index 6fc6af818f..1e4a5c1fd1 100644 --- a/pkg/ocm/storage/received/upload.go +++ b/pkg/ocm/storage/received/upload.go @@ -77,7 +77,20 @@ func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, up } func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) { - client, _, rel, err := d.webdavClient(ctx, nil, req.Ref) + shareID, _ := shareInfoFromReference(req.Ref) + u, err := d.GetUpload(ctx, shareID.OpaqueId) + if err != nil { + return provider.ResourceInfo{}, err + } + + info, err := u.GetInfo(ctx) + if err != nil { + return provider.ResourceInfo{}, err + } + + client, _, rel, err := d.webdavClient(ctx, nil, &provider.Reference{ + Path: filepath.Join(info.MetaData["dir"], info.MetaData["filename"]), + }) if err != nil { return provider.ResourceInfo{}, err }