Handle non-TUS uploads called directly with upload path (#1314)

changelog/unreleased/uploads-refactor.md

@@ -6,3 +6,4 @@ refactors that workflow to accept incoming requests following the tus protocol
 while using simpler transmission internally.
 
 https://github.com/cs3org/reva/pull/1285
+https://github.com/cs3org/reva/pull/1314

internal/http/services/owncloud/ocdav/get.go

@@ -142,7 +142,7 @@ func (s *svc) handleGet(w http.ResponseWriter, r *http.Request, ns string) {
 	w.Header().Set("ETag", info.Etag)
 	w.Header().Set("OC-FileId", wrapResourceID(info.Id))
 	w.Header().Set("OC-ETag", info.Etag)
-	t := utils.TSToTime(info.Mtime)
+	t := utils.TSToTime(info.Mtime).UTC()
 	lastModifiedString := t.Format(time.RFC1123Z)
 	w.Header().Set("Last-Modified", lastModifiedString)
 	w.Header().Set("Content-Length", strconv.FormatUint(info.Size, 10))

internal/http/services/owncloud/ocdav/head.go

@@ -76,7 +76,7 @@ func (s *svc) handleHead(w http.ResponseWriter, r *http.Request, ns string) {
 	w.Header().Set("ETag", info.Etag)
 	w.Header().Set("OC-FileId", wrapResourceID(info.Id))
 	w.Header().Set("OC-ETag", info.Etag)
-	t := utils.TSToTime(info.Mtime)
+	t := utils.TSToTime(info.Mtime).UTC()
 	lastModifiedString := t.Format(time.RFC1123Z)
 	w.Header().Set("Last-Modified", lastModifiedString)
 	w.WriteHeader(http.StatusOK)

internal/http/services/owncloud/ocdav/put.go

@@ -312,7 +312,7 @@ func (s *svc) handlePutHelper(w http.ResponseWriter, r *http.Request, content io
 	w.Header().Set("ETag", newInfo.Etag)
 	w.Header().Set("OC-FileId", wrapResourceID(newInfo.Id))
 	w.Header().Set("OC-ETag", newInfo.Etag)
-	t := utils.TSToTime(newInfo.Mtime)
+	t := utils.TSToTime(newInfo.Mtime).UTC()
 	lastModifiedString := t.Format(time.RFC1123Z)
 	w.Header().Set("Last-Modified", lastModifiedString)
 

internal/http/services/owncloud/ocdav/tus.go

@@ -264,7 +264,7 @@ func (s *svc) handleTusPost(w http.ResponseWriter, r *http.Request, ns string) {
 			w.Header().Set("OC-FileId", wrapResourceID(info.Id))
 			w.Header().Set("OC-ETag", info.Etag)
 			w.Header().Set("ETag", info.Etag)
-			t := utils.TSToTime(info.Mtime)
+			t := utils.TSToTime(info.Mtime).UTC()
 			lastModifiedString := t.Format(time.RFC1123Z)
 			w.Header().Set("Last-Modified", lastModifiedString)
 		}

pkg/storage/fs/ocis/upload.go

@@ -44,15 +44,26 @@ var defaultFilePerm = os.FileMode(0664)
 func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) (err error) {
 	upload, err := fs.GetUpload(ctx, ref.GetPath())
 	if err != nil {
-		return errors.Wrap(err, "ocisfs: error retrieving upload")
+		// Upload corresponding to this ID was not found.
+		// Assume that this corresponds to the resource path to which the file has to be uploaded.
+
+		// Set the length to 0 and set SizeIsDeferred to true
+		metadata := map[string]string{"sizedeferred": "true"}
+		uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata)
+		if err != nil {
+			return err
+		}
+		if upload, err = fs.GetUpload(ctx, uploadID); err != nil {
+			return errors.Wrap(err, "ocisfs: error retrieving upload")
+		}
 	}
 
 	uploadInfo := upload.(*fileUpload)
 
 	p := uploadInfo.info.Storage["NodeName"]
 	ok, err := chunking.IsChunked(p)
 	if err != nil {
-		return errors.Wrap(err, "ocfs: error checking path")
+		return errors.Wrap(err, "ocisfs: error checking path")
 	}
 	if ok {
 		var assembledFile string
@@ -69,7 +80,7 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read
 		uploadInfo.info.Storage["NodeName"] = p
 		fd, err := os.Open(assembledFile)
 		if err != nil {
-			return errors.Wrap(err, "eos: error opening assembled file")
+			return errors.Wrap(err, "ocisfs: error opening assembled file")
 		}
 		defer fd.Close()
 		defer os.RemoveAll(assembledFile)
@@ -111,8 +122,13 @@ func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, u
 		Size: uploadLength,
 	}
 
-	if metadata != nil && metadata["mtime"] != "" {
-		info.MetaData["mtime"] = metadata["mtime"]
+	if metadata != nil {
+		if metadata["mtime"] != "" {
+			info.MetaData["mtime"] = metadata["mtime"]
+		}
+		if _, ok := metadata["sizedeferred"]; ok {
+			info.SizeIsDeferred = true
+		}
 	}
 
 	log.Debug().Interface("info", info).Interface("node", n).Interface("metadata", metadata).Msg("ocisfs: resolved filename")

pkg/storage/fs/owncloud/upload.go

@@ -44,7 +44,18 @@ var defaultFilePerm = os.FileMode(0664)
 func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error {
 	upload, err := fs.GetUpload(ctx, ref.GetPath())
 	if err != nil {
-		return errors.Wrap(err, "ocfs: error retrieving upload")
+		// Upload corresponding to this ID was not found.
+		// Assume that this corresponds to the resource path to which the file has to be uploaded.
+
+		// Set the length to 0 and set SizeIsDeferred to true
+		metadata := map[string]string{"sizedeferred": "true"}
+		uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata)
+		if err != nil {
+			return err
+		}
+		if upload, err = fs.GetUpload(ctx, uploadID); err != nil {
+			return errors.Wrap(err, "ocfs: error retrieving upload")
+		}
 	}
 
 	uploadInfo := upload.(*fileUpload)
@@ -69,7 +80,7 @@ func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCl
 		uploadInfo.info.Storage["InternalDestination"] = p
 		fd, err := os.Open(assembledFile)
 		if err != nil {
-			return errors.Wrap(err, "eos: error opening assembled file")
+			return errors.Wrap(err, "ocfs: error opening assembled file")
 		}
 		defer fd.Close()
 		defer os.RemoveAll(assembledFile)
@@ -103,8 +114,13 @@ func (fs *ocfs) InitiateUpload(ctx context.Context, ref *provider.Reference, upl
 		Size: uploadLength,
 	}
 
-	if metadata != nil && metadata["mtime"] != "" {
-		info.MetaData["mtime"] = metadata["mtime"]
+	if metadata != nil {
+		if metadata["mtime"] != "" {
+			info.MetaData["mtime"] = metadata["mtime"]
+		}
+		if _, ok := metadata["sizedeferred"]; ok {
+			info.SizeIsDeferred = true
+		}
 	}
 
 	upload, err := fs.NewUpload(ctx, info)

pkg/storage/utils/localfs/upload.go

@@ -42,15 +42,26 @@ var defaultFilePerm = os.FileMode(0664)
 func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error {
 	upload, err := fs.GetUpload(ctx, ref.GetPath())
 	if err != nil {
-		return errors.Wrap(err, "ocisfs: error retrieving upload")
+		// Upload corresponding to this ID was not found.
+		// Assume that this corresponds to the resource path to which the file has to be uploaded.
+
+		// Set the length to 0 and set SizeIsDeferred to true
+		metadata := map[string]string{"sizedeferred": "true"}
+		uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata)
+		if err != nil {
+			return err
+		}
+		if upload, err = fs.GetUpload(ctx, uploadID); err != nil {
+			return errors.Wrap(err, "localfs: error retrieving upload")
+		}
 	}
 
 	uploadInfo := upload.(*fileUpload)
 
 	p := uploadInfo.info.Storage["InternalDestination"]
 	ok, err := chunking.IsChunked(p)
 	if err != nil {
-		return errors.Wrap(err, "ocfs: error checking path")
+		return errors.Wrap(err, "localfs: error checking path")
 	}
 	if ok {
 		var assembledFile string
@@ -60,22 +71,22 @@ func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.Rea
 		}
 		if p == "" {
 			if err = uploadInfo.Terminate(ctx); err != nil {
-				return errors.Wrap(err, "ocfs: error removing auxiliary files")
+				return errors.Wrap(err, "localfs: error removing auxiliary files")
 			}
 			return errtypes.PartialContent(ref.String())
 		}
 		uploadInfo.info.Storage["InternalDestination"] = p
 		fd, err := os.Open(assembledFile)
 		if err != nil {
-			return errors.Wrap(err, "eos: error opening assembled file")
+			return errors.Wrap(err, "localfs: error opening assembled file")
 		}
 		defer fd.Close()
 		defer os.RemoveAll(assembledFile)
 		r = fd
 	}
 
 	if _, err := uploadInfo.WriteChunk(ctx, 0, r); err != nil {
-		return errors.Wrap(err, "ocisfs: error writing to binary file")
+		return errors.Wrap(err, "localfs: error writing to binary file")
 	}
 
 	return uploadInfo.FinishUpload(ctx)
@@ -101,8 +112,13 @@ func (fs *localfs) InitiateUpload(ctx context.Context, ref *provider.Reference,
 		Size: uploadLength,
 	}
 
-	if metadata != nil && metadata["mtime"] != "" {
-		info.MetaData["mtime"] = metadata["mtime"]
+	if metadata != nil {
+		if metadata["mtime"] != "" {
+			info.MetaData["mtime"] = metadata["mtime"]
+		}
+		if _, ok := metadata["sizedeferred"]; ok {
+			info.SizeIsDeferred = true
+		}
 	}
 
 	upload, err := fs.NewUpload(ctx, info)
@@ -331,8 +347,10 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error {
 
 	// only delete the upload if it was successfully written to the fs
 	if err := os.Remove(upload.infoPath); err != nil {
-		log := appctx.GetLogger(ctx)
-		log.Err(err).Interface("info", upload.info).Msg("localfs: could not delete upload info")
+		if !os.IsNotExist(err) {
+			log := appctx.GetLogger(ctx)
+			log.Err(err).Interface("info", upload.info).Msg("localfs: could not delete upload info")
+		}
 	}
 
 	// TODO: set mtime if specified in metadata