diff --git a/changelog/unreleased/ocdav-putchunked.md b/changelog/unreleased/ocdav-putchunked.md new file mode 100644 index 0000000000..69e50bf424 --- /dev/null +++ b/changelog/unreleased/ocdav-putchunked.md @@ -0,0 +1,7 @@ +Bugfix: Upload file to storage provider after assembling chunks + +In the PUT handler for chunked uploads in ocdav, we store the individual +chunks in temporary file but do not write the assembled file to storage. +This PR fixes that. + +https://github.com/cs3org/reva/pull/1253 diff --git a/cmd/reva/main.go b/cmd/reva/main.go index 56e9d9fdff..e726959fd2 100644 --- a/cmd/reva/main.go +++ b/cmd/reva/main.go @@ -21,10 +21,13 @@ package main import ( "flag" "fmt" + "net/http" "os" "strings" + "time" "github.com/c-bata/go-prompt" + "github.com/cs3org/reva/pkg/rhttp" ) var ( @@ -37,6 +40,8 @@ var ( gitCommit, buildDate, version, goVersion string + client *http.Client + commands = []*command{ versionCommand(), configureCommand(), @@ -94,6 +99,13 @@ func main() { } } + client = rhttp.GetHTTPClient( + // TODO make insecure configurable + rhttp.Insecure(true), + // TODO make timeout configurable + rhttp.Timeout(time.Duration(24*int64(time.Hour))), + ) + generateMainUsage() executor := Executor{Timeout: timeout} completer := Completer{DisableArgPrompt: disableargprompt} diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index ddcdbe9ee0..e3929a3b4a 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -26,12 +26,10 @@ import ( "os" "path/filepath" "strconv" - "time" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/pkg/errors" - "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" @@ -84,7 +82,6 @@ func uploadCommand() *command { if err != nil { return err } - defer fd.Close() fmt.Printf("Local file size: %d bytes\n", md.Size()) @@ -149,14 +146,9 @@ func uploadCommand() *command { dataServerURL := res.UploadEndpoint - bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) - bar.Start() - reader := bar.NewProxyReader(fd) - if *disableTusFlag { - httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader) + httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, fd) if err != nil { - bar.Finish() return err } @@ -166,38 +158,21 @@ func uploadCommand() *command { q.Add("xs_type", storageprovider.GRPC2PKGXS(xsType).String()) httpReq.URL.RawQuery = q.Encode() - httpClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - // TODO make insecure configurable - rhttp.Insecure(true), - // TODO make timeout configurable - rhttp.Timeout(time.Duration(24*int64(time.Hour))), - ) - - httpRes, err := httpClient.Do(httpReq) + httpRes, err := client.Do(httpReq) if err != nil { - bar.Finish() return err } defer httpRes.Body.Close() if httpRes.StatusCode != http.StatusOK { - bar.Finish() return err } } else { // create the tus client. c := tus.DefaultConfig() c.Resume = true - c.HttpClient = rhttp.GetHTTPClient( - rhttp.Context(ctx), - // TODO make insecure configurable - rhttp.Insecure(true), - // TODO make timeout configurable - rhttp.Timeout(time.Duration(24*int64(time.Hour))), - ) + c.HttpClient = client c.Store, err = memorystore.NewMemoryStore() if err != nil { - bar.Finish() return err } if token, ok := tokenpkg.ContextGetToken(ctx); ok { @@ -208,7 +183,6 @@ func uploadCommand() *command { } tusc, err := tus.NewClient(dataServerURL, c) if err != nil { - bar.Finish() return err } @@ -221,7 +195,7 @@ func uploadCommand() *command { fingerprint := fmt.Sprintf("%s-%d-%s-%s", md.Name(), md.Size(), md.ModTime(), xs) // create an upload from a file. - upload := tus.NewUpload(reader, md.Size(), metadata, fingerprint) + upload := tus.NewUpload(fd, md.Size(), metadata, fingerprint) // create the uploader. c.Store.Set(upload.Fingerprint, dataServerURL) @@ -230,13 +204,10 @@ func uploadCommand() *command { // start the uploading process. err = uploader.Upload() if err != nil { - bar.Finish() return err } } - bar.Finish() - req2 := &provider.StatRequest{ Ref: &provider.Reference{ Spec: &provider.Reference_Path{ @@ -291,21 +262,15 @@ func checkUploadWebdavRef(endpoint string, opaque *typespb.Opaque, md os.FileInf return errors.New("opaque entry decoder not recognized: " + fileOpaque.Decoder) } - bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) - bar.Start() - reader := bar.NewProxyReader(fd) - c := gowebdav.NewClient(endpoint, "", "") c.SetHeader(tokenpkg.TokenHeader, token) c.SetHeader("Upload-Length", strconv.FormatInt(md.Size(), 10)) - err := c.WriteStream(filePath, reader, 0700) + err := c.WriteStream(filePath, fd, 0700) if err != nil { - bar.Finish() return err } - bar.Finish() fmt.Println("File uploaded") return nil } diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 7a2a55c393..c6284af2e0 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -50,6 +50,7 @@ func init() { type service struct { provider app.Provider + client *http.Client conf *config } @@ -79,6 +80,9 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, provider: provider, + client: rhttp.GetHTTPClient( + rhttp.Timeout(5 * time.Second), + ), } return service, nil @@ -120,12 +124,6 @@ func getProvider(c *config) (app.Provider, error) { } func (s *service) getWopiAppEndpoints(ctx context.Context) (map[string]interface{}, error) { - httpClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - // calls to WOPI are expected to take a very short time, 5s (though hardcoded) ought to be far enough - rhttp.Timeout(time.Duration(5*int64(time.Second))), - ) - // TODO this query will eventually be served by Reva. // For the time being it is a remnant of the CERNBox-specific WOPI server, which justifies the /cbox path in the URL. wopiurl, err := url.Parse(s.conf.WopiURL) @@ -137,7 +135,7 @@ func (s *service) getWopiAppEndpoints(ctx context.Context) (map[string]interface if err != nil { return nil, err } - appsRes, err := httpClient.Do(appsReq) + appsRes, err := s.client.Do(appsReq) if err != nil { return nil, err } @@ -165,12 +163,6 @@ func (s *service) OpenFileInAppProvider(ctx context.Context, req *providerpb.Ope log := appctx.GetLogger(ctx) - httpClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - // calls to WOPI are expected to take a very short time, 5s (though hardcoded) ought to be far enough - rhttp.Timeout(time.Duration(5*int64(time.Second))), - ) - wopiurl, err := url.Parse(s.conf.WopiURL) if err != nil { return nil, err @@ -203,7 +195,7 @@ func (s *service) OpenFileInAppProvider(ctx context.Context, req *providerpb.Ope httpReq.URL.RawQuery = q.Encode() - openRes, err := httpClient.Do(httpReq) + openRes, err := s.client.Do(httpReq) if err != nil { res := &providerpb.OpenFileInAppProviderResponse{ diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index aa44ba6f40..8537e8580d 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -48,7 +48,10 @@ type transferClaims struct { } func (s *svc) sign(_ context.Context, target string) (string, error) { - ttl := time.Duration(s.c.TransferExpires) * time.Second + // Tus sends a separate request to the datagateway service for every chunk. + // For large files, this can take a long time, so we extend the expiration + // for 10 minutes. TODO: Make this configurable. + ttl := time.Duration(s.c.TransferExpires) * 10 * time.Minute claims := transferClaims{ StandardClaims: jwt.StandardClaims{ ExpiresAt: time.Now().Add(ttl).Unix(), diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 46a472fb0f..7b62ea64d9 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -36,12 +36,13 @@ func init() { } type config struct { - Prefix string `mapstructure:"prefix" docs:"data;The prefix to be used for this HTTP service"` - Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` - Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go;The configuration for the storage driver"` - Timeout int64 `mapstructure:"timeout"` - Insecure bool `mapstructure:"insecure"` - DisableTus bool `mapstructure:"disable_tus" docs:"false;Whether to disable TUS uploads."` + Prefix string `mapstructure:"prefix" docs:"data;The prefix to be used for this HTTP service"` + Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` + Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go;The configuration for the storage driver"` + Timeout int64 `mapstructure:"timeout"` + Insecure bool `mapstructure:"insecure"` + DisableTus bool `mapstructure:"disable_tus" docs:"false;Whether to disable TUS uploads."` + TempDirectory string `mapstructure:"temp_directory"` } func (c *config) init() { @@ -53,6 +54,10 @@ func (c *config) init() { c.Driver = "localhome" } + if c.TempDirectory == "" { + c.TempDirectory = "/var/tmp/reva/tmp" + } + } type svc struct { diff --git a/internal/http/services/dataprovider/put.go b/internal/http/services/dataprovider/put.go index 1bcbfbb65b..13e957688e 100644 --- a/internal/http/services/dataprovider/put.go +++ b/internal/http/services/dataprovider/put.go @@ -20,7 +20,10 @@ package dataprovider import ( "fmt" + "io" + "io/ioutil" "net/http" + "os" "path" "strconv" "strings" @@ -73,6 +76,18 @@ func (s *svc) doTusPut(w http.ResponseWriter, r *http.Request) { } } + fd, err := ioutil.TempFile(fmt.Sprintf("/%s", s.conf.TempDirectory), "") + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + defer os.RemoveAll(fd.Name()) + defer fd.Close() + if _, err := io.Copy(fd, r.Body); err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + dataServerURL := fmt.Sprintf("http://%s%s", r.Host, r.RequestURI) // create the tus client. @@ -102,7 +117,7 @@ func (s *svc) doTusPut(w http.ResponseWriter, r *http.Request) { "dir": path.Dir(fp), } - upload := tus.NewUpload(r.Body, length, metadata, "") + upload := tus.NewUpload(fd, length, metadata, "") defer r.Body.Close() // create the uploader. diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index 032ff70841..631d002120 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "net/http" + "os" "path" "strings" @@ -293,8 +294,18 @@ func (s *svc) descend(ctx context.Context, client gateway.GatewayAPIClient, src return fmt.Errorf("status code %d", httpDownloadRes.StatusCode) } + fileName, fd, err := s.createChunkTempFile() + if err != nil { + return err + } + defer os.RemoveAll(fileName) + defer fd.Close() + if _, err := io.Copy(fd, httpDownloadRes.Body); err != nil { + return err + } + // do upload - err = s.tusUpload(ctx, uRes.UploadEndpoint, uRes.Token, dst, httpDownloadRes.Body, src.GetSize()) + err = s.tusUpload(ctx, uRes.UploadEndpoint, uRes.Token, dst, fd, int64(src.GetSize())) if err != nil { return err } @@ -302,7 +313,7 @@ func (s *svc) descend(ctx context.Context, client gateway.GatewayAPIClient, src return nil } -func (s *svc) tusUpload(ctx context.Context, dataServerURL string, transferToken string, fn string, body io.Reader, length uint64) error { +func (s *svc) tusUpload(ctx context.Context, dataServerURL string, transferToken string, fn string, body io.ReadSeeker, length int64) error { var err error log := appctx.GetLogger(ctx) @@ -339,7 +350,7 @@ func (s *svc) tusUpload(ctx context.Context, dataServerURL string, transferToken Str("dir", path.Dir(fn)). Msg("tus.NewUpload") - upload := tus.NewUpload(body, int64(length), metadata, "") + upload := tus.NewUpload(body, length, metadata, "") // create the uploader. c.Store.Set(upload.Fingerprint, dataServerURL) diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 3571223600..1517e3a803 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -29,9 +29,9 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/utils" ) func (s *svc) handleGet(w http.ResponseWriter, r *http.Request, ns string) { diff --git a/internal/http/services/owncloud/ocdav/head.go b/internal/http/services/owncloud/ocdav/head.go index 8760d37d39..02e776b560 100644 --- a/internal/http/services/owncloud/ocdav/head.go +++ b/internal/http/services/owncloud/ocdav/head.go @@ -26,8 +26,8 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/utils" ) func (s *svc) handleHead(w http.ResponseWriter, r *http.Request, ns string) { diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index 3b18e2db17..f570315087 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -36,8 +36,8 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" ) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 4310d8789c..9ac9c40878 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -19,7 +19,9 @@ package ocdav import ( + "io" "net/http" + "os" "path" "regexp" "strconv" @@ -28,12 +30,8 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/internal/http/services/datagateway" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" - tokenpkg "github.com/cs3org/reva/pkg/token" - "github.com/eventials/go-tus" - "github.com/eventials/go-tus/memorystore" + "github.com/cs3org/reva/pkg/utils" ) func isChunked(fn string) (bool, error) { @@ -137,7 +135,7 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { w.WriteHeader(http.StatusBadRequest) } */ - s.handlePutChunked(w, r) + s.handlePutChunked(w, r, ns) return } @@ -156,6 +154,34 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } } + length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) + if err != nil { + // Fallback to Upload-Length + length, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + } + fileName, fd, err := s.createChunkTempFile() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + defer os.RemoveAll(fileName) + defer fd.Close() + if _, err := io.Copy(fd, r.Body); err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + s.handlePutHelper(w, r, fd, fn, length) +} + +func (s *svc) handlePutHelper(w http.ResponseWriter, r *http.Request, content io.ReadSeeker, fn string, length int64) { + ctx := r.Context() + log := appctx.GetLogger(ctx) + client, err := s.getClient() if err != nil { log.Error().Err(err).Msg("error getting grpc client") @@ -163,11 +189,10 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { return } - sReq := &provider.StatRequest{ - Ref: &provider.Reference{ - Spec: &provider.Reference_Path{Path: fn}, - }, + ref := &provider.Reference{ + Spec: &provider.Reference_Path{Path: fn}, } + sReq := &provider.StatRequest{Ref: ref} sRes, err := client.Stat(ctx, sReq) if err != nil { log.Error().Err(err).Msg("error sending grpc stat request") @@ -206,16 +231,6 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } } - length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) - if err != nil { - // Fallback to Upload-Length - length, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - } - opaqueMap := map[string]*typespb.OpaqueEntry{ "Upload-Length": { Decoder: "plain", @@ -235,9 +250,7 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } uReq := &provider.InitiateFileUploadRequest{ - Ref: &provider.Reference{ - Spec: &provider.Reference_Path{Path: fn}, - }, + Ref: ref, Opaque: &typespb.Opaque{ Map: opaqueMap, }, @@ -266,52 +279,8 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { return } - dataServerURL := uRes.UploadEndpoint - - // create the tus client. - c := tus.DefaultConfig() - c.Resume = true - c.HttpClient = s.client - - c.Store, err = memorystore.NewMemoryStore() - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - - log.Debug(). - Str("upload-endpoint", dataServerURL). - Str("auth-header", tokenpkg.TokenHeader). - Str("auth-token", tokenpkg.ContextMustGetToken(ctx)). - Str("transfer-header", datagateway.TokenTransportHeader). - Str("transfer-token", uRes.Token). - Msg("adding tokens to headers") - c.Header.Set(tokenpkg.TokenHeader, tokenpkg.ContextMustGetToken(ctx)) - c.Header.Set(datagateway.TokenTransportHeader, uRes.Token) - - tusc, err := tus.NewClient(dataServerURL, c) - if err != nil { - log.Error().Err(err).Msg("Could not get TUS client") - w.WriteHeader(http.StatusInternalServerError) - return - } - - metadata := map[string]string{ - "filename": path.Base(fn), - "dir": path.Dir(fn), - //"checksum": fmt.Sprintf("%s %s", storageprovider.GRPC2PKGXS(xsType).String(), xs), - } - - upload := tus.NewUpload(r.Body, length, metadata, "") - - // create the uploader. - c.Store.Set(upload.Fingerprint, dataServerURL) - uploader := tus.NewUploader(tusc, dataServerURL, upload, 0) - - // start the uploading process. - err = uploader.Upload() - if err != nil { - log.Error().Err(err).Msg("Could not start TUS upload") + if err = s.tusUpload(ctx, uRes.UploadEndpoint, uRes.Token, fn, content, length); err != nil { + log.Error().Err(err).Msg("TUS upload failed") w.WriteHeader(http.StatusInternalServerError) return } diff --git a/internal/http/services/owncloud/ocdav/putchunked.go b/internal/http/services/owncloud/ocdav/putchunked.go index 80ec093edf..2708643d5f 100644 --- a/internal/http/services/owncloud/ocdav/putchunked.go +++ b/internal/http/services/owncloud/ocdav/putchunked.go @@ -25,12 +25,11 @@ import ( "io/ioutil" "net/http" "os" + "path" "path/filepath" "strconv" "strings" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ) @@ -215,10 +214,9 @@ func (s *svc) saveChunk(ctx context.Context, path string, r io.ReadCloser) (bool return false, "", err } - tempFileName := assembledFileName - return true, tempFileName, nil + return true, assembledFileName, nil } -func (s *svc) handlePutChunked(w http.ResponseWriter, r *http.Request) { +func (s *svc) handlePutChunked(w http.ResponseWriter, r *http.Request, ns string) { ctx := r.Context() log := appctx.GetLogger(ctx) @@ -250,57 +248,17 @@ func (s *svc) handlePutChunked(w http.ResponseWriter, r *http.Request) { } defer fd.Close() - chunkInfo, _ := getChunkBLOBInfo(fn) - - client, err := s.getClient() - if err != nil { - log.Error().Err(err).Msg("error getting grpc client") - w.WriteHeader(http.StatusInternalServerError) - return - } - - ref := &provider.Reference{ - Spec: &provider.Reference_Path{Path: chunkInfo.path}, - } - req := &provider.StatRequest{Ref: ref} - res, err := client.Stat(ctx, req) + md, err := fd.Stat() if err != nil { - log.Error().Err(err).Msg("error sending grpc stat request") + log.Error().Err(err).Msg("error statting chunk") w.WriteHeader(http.StatusInternalServerError) return } - if res.Status.Code != rpc.Code_CODE_OK && res.Status.Code != rpc.Code_CODE_NOT_FOUND { - switch res.Status.Code { - case rpc.Code_CODE_PERMISSION_DENIED: - log.Debug().Str("path", fn).Interface("status", res.Status).Msg("permission denied") - w.WriteHeader(http.StatusForbidden) - default: - log.Error().Str("path", fn).Interface("status", res.Status).Msg("grpc stat request failed") - w.WriteHeader(http.StatusInternalServerError) - } - return - } - - info := res.Info - if info != nil && info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { - log.Warn().Msg("resource is not a file") - w.WriteHeader(http.StatusConflict) - return - } + chunkInfo, _ := getChunkBLOBInfo(fn) + fn = path.Join(applyLayout(ctx, ns), chunkInfo.path) - if info != nil { - clientETag := r.Header.Get("If-Match") - serverETag := info.Etag - if clientETag != "" { - if clientETag != serverETag { - log.Warn().Str("client-etag", clientETag).Str("server-etag", serverETag).Msg("etags mismatch") - w.WriteHeader(http.StatusPreconditionFailed) - return - } - } - } - w.WriteHeader(http.StatusCreated) + s.handlePutHelper(w, r, fd, fn, md.Size()) // TODO(labkode): implement old chunking diff --git a/internal/http/services/owncloud/ocdav/trashbin.go b/internal/http/services/owncloud/ocdav/trashbin.go index 4a43afa3d5..6386d01643 100644 --- a/internal/http/services/owncloud/ocdav/trashbin.go +++ b/internal/http/services/owncloud/ocdav/trashbin.go @@ -32,11 +32,11 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" ctxuser "github.com/cs3org/reva/pkg/user" + "github.com/cs3org/reva/pkg/utils" ) // TrashbinHandler handles trashbin requests diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index e3dc016948..d9319cc051 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -28,9 +28,9 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/utils" tusd "github.com/tus/tusd/pkg/handler" ) diff --git a/internal/http/utils/utils.go b/internal/http/utils/utils.go deleted file mode 100644 index 2a74ab1ed6..0000000000 --- a/internal/http/utils/utils.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package utils - -import ( - "time" - - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" -) - -// TSToUnixNano converts a protobuf Timestamp to uint64 -// with nanoseconds resolution. -func TSToUnixNano(ts *types.Timestamp) uint64 { - return uint64(time.Unix(int64(ts.Seconds), int64(ts.Nanos)).UnixNano()) -} - -// TSToTime converts a protobuf Timestamp to Go's time.Time. -func TSToTime(ts *types.Timestamp) time.Time { - return time.Unix(int64(ts.Seconds), int64(ts.Nanos)) -} diff --git a/pkg/auth/manager/json/json.go b/pkg/auth/manager/json/json.go index 377ba6ddad..5f5b2f69be 100644 --- a/pkg/auth/manager/json/json.go +++ b/pkg/auth/manager/json/json.go @@ -24,6 +24,7 @@ import ( "io/ioutil" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/errtypes" @@ -37,13 +38,14 @@ func init() { // Credentials holds a pair of secret and userid type Credentials struct { - ID *user.UserId `mapstructure:"id" json:"id"` - Username string `mapstructure:"username" json:"username"` - Mail string `mapstructure:"mail" json:"mail"` - MailVerified bool `mapstructure:"mail_verified" json:"mail_verified"` - DisplayName string `mapstructure:"display_name" json:"display_name"` - Secret string `mapstructure:"secret" json:"secret"` - Groups []string `mapstructure:"groups" json:"groups"` + ID *user.UserId `mapstructure:"id" json:"id"` + Username string `mapstructure:"username" json:"username"` + Mail string `mapstructure:"mail" json:"mail"` + MailVerified bool `mapstructure:"mail_verified" json:"mail_verified"` + DisplayName string `mapstructure:"display_name" json:"display_name"` + Secret string `mapstructure:"secret" json:"secret"` + Groups []string `mapstructure:"groups" json:"groups"` + Opaque *typespb.Opaque `mapstructure:"opaque" json:"opaque"` } type manager struct { @@ -109,6 +111,7 @@ func (m *manager) Authenticate(ctx context.Context, username string, secret stri MailVerified: c.MailVerified, DisplayName: c.DisplayName, Groups: c.Groups, + Opaque: c.Opaque, // TODO add arbitrary keys as opaque data }, nil } diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index ab87ff00b8..ba4daf1108 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -552,9 +552,10 @@ func (c *Client) List(ctx context.Context, uid, gid, path string) ([]*FileInfo, // Read reads a file from the mgm func (c *Client) Read(ctx context.Context, uid, gid, path string) (io.ReadCloser, error) { - uuid := uuid.Must(uuid.NewV4()) - rand := "eosread-" + uuid.String() + rand := "eosread-" + uuid.Must(uuid.NewV4()).String() localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) + defer os.RemoveAll(localTarget) + xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", xrdPath, localTarget, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s", uid, gid)) _, _, err := c.execute(ctx, cmd) diff --git a/pkg/eosclientgrpc/eosclientgrpc.go b/pkg/eosclientgrpc/eosclientgrpc.go index def76bac9a..0df6b8a949 100644 --- a/pkg/eosclientgrpc/eosclientgrpc.go +++ b/pkg/eosclientgrpc/eosclientgrpc.go @@ -997,9 +997,10 @@ func (c *Client) Read(ctx context.Context, username, path string) (io.ReadCloser if err != nil { return nil, err } - uuid := uuid.Must(uuid.NewV4()) - rand := "eosread-" + uuid.String() + rand := "eosread-" + uuid.Must(uuid.NewV4()).String() localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) + defer os.RemoveAll(localTarget) + xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", xrdPath, localTarget, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) _, _, err = c.execute(ctx, cmd) diff --git a/pkg/ocm/invite/manager/json/json.go b/pkg/ocm/invite/manager/json/json.go index c7bfda7462..c82d04ed88 100644 --- a/pkg/ocm/invite/manager/json/json.go +++ b/pkg/ocm/invite/manager/json/json.go @@ -56,6 +56,7 @@ type manager struct { config *config sync.Mutex // concurrent access to the file model *inviteModel + client *http.Client } type config struct { @@ -103,6 +104,10 @@ func New(m map[string]interface{}) (invite.Manager, error) { manager := &manager{ config: config, model: model, + client: rhttp.GetHTTPClient( + rhttp.Timeout(5*time.Second), + rhttp.Insecure(config.InsecureConnections), + ), } return manager, nil @@ -214,15 +219,13 @@ func (m *manager) ForwardInvite(ctx context.Context, invite *invitepb.InviteToke u.Path = path.Join(u.Path, acceptInviteEndpoint) recipientURL := u.String() - client := rhttp.GetHTTPClient(rhttp.Insecure(m.config.InsecureConnections)) - req, err := http.NewRequest("POST", recipientURL, strings.NewReader(requestBody.Encode())) if err != nil { return errors.Wrap(err, "json: error framing post request") } req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - resp, err := client.Do(req) + resp, err := m.client.Do(req) if err != nil { err = errors.Wrap(err, "json: error sending post request") return err diff --git a/pkg/ocm/invite/manager/memory/memory.go b/pkg/ocm/invite/manager/memory/memory.go index 3bdada8ba1..0dfc4c2822 100644 --- a/pkg/ocm/invite/manager/memory/memory.go +++ b/pkg/ocm/invite/manager/memory/memory.go @@ -66,12 +66,17 @@ func New(m map[string]interface{}) (invite.Manager, error) { Invites: sync.Map{}, AcceptedUsers: sync.Map{}, Config: c, + Client: rhttp.GetHTTPClient( + rhttp.Timeout(5*time.Second), + rhttp.Insecure(c.InsecureConnections), + ), }, nil } type manager struct { Invites sync.Map AcceptedUsers sync.Map + Client *http.Client Config *config } @@ -114,15 +119,13 @@ func (m *manager) ForwardInvite(ctx context.Context, invite *invitepb.InviteToke u.Path = path.Join(u.Path, acceptInviteEndpoint) recipientURL := u.String() - client := rhttp.GetHTTPClient(rhttp.Insecure(m.Config.InsecureConnections)) - req, err := http.NewRequest("POST", recipientURL, strings.NewReader(requestBody.Encode())) if err != nil { return errors.Wrap(err, "json: error framing post request") } req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - resp, err := client.Do(req) + resp, err := m.Client.Do(req) if err != nil { err = errors.Wrap(err, "memory: error sending post request") return err diff --git a/pkg/ocm/share/manager/json/json.go b/pkg/ocm/share/manager/json/json.go index b1a5768fad..8eda0ebe5f 100644 --- a/pkg/ocm/share/manager/json/json.go +++ b/pkg/ocm/share/manager/json/json.go @@ -73,6 +73,9 @@ func New(m map[string]interface{}) (share.Manager, error) { mgr := &mgr{ c: c, model: model, + client: rhttp.GetHTTPClient( + rhttp.Timeout(5 * time.Second), + ), } return mgr, nil @@ -136,6 +139,7 @@ type mgr struct { c *config sync.Mutex // concurrent access to the file model *shareModel + client *http.Client } func (m *shareModel) Save() error { @@ -299,15 +303,13 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr u.Path = path.Join(u.Path, createOCMCoreShareEndpoint) recipientURL := u.String() - client := rhttp.GetHTTPClient(rhttp.Insecure(m.c.InsecureConnections)) - req, err := http.NewRequest("POST", recipientURL, strings.NewReader(requestBody.Encode())) if err != nil { return nil, errors.Wrap(err, "json: error framing post request") } req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - resp, err := client.Do(req) + resp, err := m.client.Do(req) if err != nil { err = errors.Wrap(err, "json: error sending post request") return nil, err diff --git a/pkg/ocm/share/manager/memory/memory.go b/pkg/ocm/share/manager/memory/memory.go index 83137e07e6..7548694ce8 100644 --- a/pkg/ocm/share/manager/memory/memory.go +++ b/pkg/ocm/share/manager/memory/memory.go @@ -65,6 +65,9 @@ func New(m map[string]interface{}) (share.Manager, error) { c: c, shares: sync.Map{}, state: state, + client: rhttp.GetHTTPClient( + rhttp.Timeout(5 * time.Second), + ), }, nil } @@ -72,6 +75,7 @@ type mgr struct { c *config shares sync.Map state map[string]map[string]ocm.ShareState + client *http.Client } type config struct { @@ -209,15 +213,13 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr u.Path = path.Join(u.Path, createOCMCoreShareEndpoint) recipientURL := u.String() - client := rhttp.GetHTTPClient(rhttp.Insecure(m.c.InsecureConnections)) - req, err := http.NewRequest("POST", recipientURL, strings.NewReader(requestBody.Encode())) if err != nil { return nil, errors.Wrap(err, "json: error framing post request") } req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - resp, err := client.Do(req) + resp, err := m.client.Do(req) if err != nil { err = errors.Wrap(err, "memory: error sending post request") return nil, err diff --git a/pkg/storage/utils/eosfs/upload.go b/pkg/storage/utils/eosfs/upload.go index 283c70b09e..39a6da9501 100644 --- a/pkg/storage/utils/eosfs/upload.go +++ b/pkg/storage/utils/eosfs/upload.go @@ -236,7 +236,12 @@ func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { // GetReader returns an io.Reader for the upload func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { - return os.Open(upload.binPath) + f, err := os.Open(upload.binPath) + if err != nil { + return nil, err + } + defer f.Close() + return f, nil } // WriteChunk writes the stream from the reader to the given offset of the upload diff --git a/pkg/user/manager/rest/rest.go b/pkg/user/manager/rest/rest.go index 5d0e7a3248..c022095ce1 100644 --- a/pkg/user/manager/rest/rest.go +++ b/pkg/user/manager/rest/rest.go @@ -54,6 +54,7 @@ type manager struct { conf *config redisPool *redis.Pool oidcToken OIDCToken + client *http.Client } // OIDCToken stores the OIDC token used to authenticate requests to the REST API service @@ -128,6 +129,10 @@ func New(m map[string]interface{}) (user.Manager, error) { return &manager{ conf: c, redisPool: redisPool, + client: rhttp.GetHTTPClient( + rhttp.Timeout(10*time.Second), + rhttp.Insecure(true), + ), }, nil } @@ -156,7 +161,6 @@ func (m *manager) getAPIToken(ctx context.Context) (string, time.Time, error) { "audience": {m.conf.TargetAPI}, } - httpClient := rhttp.GetHTTPClient(rhttp.Context(ctx), rhttp.Timeout(10*time.Second), rhttp.Insecure(true)) httpReq, err := http.NewRequest("POST", m.conf.OIDCTokenEndpoint, strings.NewReader(params.Encode())) if err != nil { return "", time.Time{}, err @@ -164,7 +168,7 @@ func (m *manager) getAPIToken(ctx context.Context) (string, time.Time, error) { httpReq.SetBasicAuth(m.conf.ClientID, m.conf.ClientSecret) httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - httpRes, err := httpClient.Do(httpReq) + httpRes, err := m.client.Do(httpReq) if err != nil { return "", time.Time{}, err } @@ -192,7 +196,6 @@ func (m *manager) sendAPIRequest(ctx context.Context, url string) ([]interface{} return nil, err } - httpClient := rhttp.GetHTTPClient(rhttp.Context(ctx), rhttp.Timeout(10*time.Second), rhttp.Insecure(true)) httpReq, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err @@ -203,7 +206,7 @@ func (m *manager) sendAPIRequest(ctx context.Context, url string) ([]interface{} // the token and expiration time while this request is in progress, the current token will still be valid. httpReq.Header.Set("Authorization", "Bearer "+m.oidcToken.apiToken) - httpRes, err := httpClient.Do(httpReq) + httpRes, err := m.client.Do(httpReq) if err != nil { return nil, err } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 427e0427cf..428725e077 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -25,6 +25,9 @@ import ( "path/filepath" "regexp" "strings" + "time" + + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ) var ( @@ -92,3 +95,14 @@ func ResolvePath(path string) (string, error) { return path, nil } + +// TSToUnixNano converts a protobuf Timestamp to uint64 +// with nanoseconds resolution. +func TSToUnixNano(ts *types.Timestamp) uint64 { + return uint64(time.Unix(int64(ts.Seconds), int64(ts.Nanos)).UnixNano()) +} + +// TSToTime converts a protobuf Timestamp to Go's time.Time. +func TSToTime(ts *types.Timestamp) time.Time { + return time.Unix(int64(ts.Seconds), int64(ts.Nanos)) +} diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.txt b/tests/acceptance/expected-failures-on-OCIS-storage.txt index 74bb5f761c..1b03f286cb 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.txt +++ b/tests/acceptance/expected-failures-on-OCIS-storage.txt @@ -1269,7 +1269,6 @@ apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:19 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:35 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:36 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:37 -apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:40 # # https://github.com/owncloud/ocis-reva/issues/56 remote.php/dav/uploads endpoint does not exist # @@ -1287,7 +1286,6 @@ apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:20 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:37 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:38 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:39 -apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:42 # # https://github.com/owncloud/ocis-reva/issues/56 remote.php/dav/uploads endpoint does not exist # @@ -1308,12 +1306,7 @@ apiWebdavUpload2/uploadFileUsingNewChunking.feature:158 # # https://github.com/owncloud/ocis-reva/issues/17 uploading with old-chunking does not work # -apiWebdavUpload2/uploadFileUsingOldChunking.feature:13 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:25 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:34 apiWebdavUpload2/uploadFileUsingOldChunking.feature:43 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:75 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:76 apiWebdavUpload2/uploadFileUsingOldChunking.feature:96 apiWebdavUpload2/uploadFileUsingOldChunking.feature:97 apiWebdavUpload2/uploadFileUsingOldChunking.feature:98 diff --git a/tests/acceptance/expected-failures-on-OWNCLOUD-storage.txt b/tests/acceptance/expected-failures-on-OWNCLOUD-storage.txt index b5bb50933d..558a989100 100644 --- a/tests/acceptance/expected-failures-on-OWNCLOUD-storage.txt +++ b/tests/acceptance/expected-failures-on-OWNCLOUD-storage.txt @@ -1250,22 +1250,15 @@ apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:19 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:35 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:36 apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:37 -apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:40 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:13 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:20 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:37 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:38 apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:39 -apiWebdavUpload2/uploadFileToExcludedDirectoryUsingOldChunking.feature:42 # # https://github.com/owncloud/ocis-reva/issues/17 uploading with old-chunking does not work # -apiWebdavUpload2/uploadFileUsingOldChunking.feature:13 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:25 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:34 apiWebdavUpload2/uploadFileUsingOldChunking.feature:43 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:75 -apiWebdavUpload2/uploadFileUsingOldChunking.feature:76 apiWebdavUpload2/uploadFileUsingOldChunking.feature:96 apiWebdavUpload2/uploadFileUsingOldChunking.feature:97 apiWebdavUpload2/uploadFileUsingOldChunking.feature:98 diff --git a/tests/acceptance/features/apiOcisSpecific/apiVersions-fileVersions.feature b/tests/acceptance/features/apiOcisSpecific/apiVersions-fileVersions.feature index c9e08b6350..a6cf291365 100644 --- a/tests/acceptance/features/apiOcisSpecific/apiVersions-fileVersions.feature +++ b/tests/acceptance/features/apiOcisSpecific/apiVersions-fileVersions.feature @@ -12,8 +12,8 @@ Feature: dav-versions Scenario: Upload file and no version is available using various chunking methods When user "Alice" uploads file "filesForUpload/davtest.txt" to filenames based on "/davtest.txt" with all mechanisms using the WebDAV API Then the version folder of file "/davtest.txt-olddav-regular" for user "Alice" should contain "0" elements - Then the version folder of file "/davtest.txt-newdav-regular" for user "Alice" should contain "0" elements - And as "Alice" file "/davtest.txt-olddav-oldchunking" should not exist + And the version folder of file "/davtest.txt-newdav-regular" for user "Alice" should contain "0" elements + And the version folder of file "/davtest.txt-olddav-oldchunking" for user "Alice" should contain "0" elements And as "Alice" file "/davtest.txt-newdav-newchunking" should not exist @issue-ocis-reva-17 @issue-ocis-reva-56 @skipOnOcis-OCIS-Storage @@ -23,7 +23,7 @@ Feature: dav-versions And user "Alice" uploads file "filesForUpload/davtest.txt" to filenames based on "/davtest.txt" with all mechanisms using the WebDAV API Then the version folder of file "/davtest.txt-olddav-regular" for user "Alice" should contain "1" element And the version folder of file "/davtest.txt-newdav-regular" for user "Alice" should contain "1" element - And as "Alice" file "/davtest.txt-olddav-oldchunking" should not exist + And the version folder of file "/davtest.txt-olddav-oldchunking" for user "Alice" should contain "1" element And as "Alice" file "/davtest.txt-newdav-newchunking" should not exist @files_sharing-app-required