From 5f9d39491d14fc90d818d398f013072b34ebe9b8 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 17 Jun 2020 16:03:58 +0200 Subject: [PATCH 1/8] Reuse transfer token const --- cmd/reva/download.go | 4 +++- cmd/reva/upload.go | 4 +++- internal/http/services/datagateway/datagateway.go | 7 ++++--- internal/http/services/owncloud/ocdav/get.go | 3 ++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/reva/download.go b/cmd/reva/download.go index 6da0e5fafd..d4d4a05f6e 100644 --- a/cmd/reva/download.go +++ b/cmd/reva/download.go @@ -24,6 +24,8 @@ import ( "net/http" "os" + "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -89,7 +91,7 @@ func downloadCommand() *command { return err } - httpReq.Header.Set("X-Reva-Transfer", res.Token) + httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token) httpClient := rhttp.GetHTTPClient(ctx) httpRes, err := httpClient.Do(httpReq) diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index 1040708ffd..a3ef054428 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -26,6 +26,8 @@ import ( "path/filepath" "strconv" + "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -134,7 +136,7 @@ func uploadCommand() *command { } if res.Token != "" { fmt.Printf("using X-Reva-Transfer header\n") - c.Header.Add("X-Reva-Transfer", res.Token) + c.Header.Add(datagateway.TokenTransportHeader, res.Token) } else if token, ok := tokenpkg.ContextGetToken(ctx); ok { fmt.Printf("using %s header\n", tokenpkg.TokenHeader) c.Header.Add(tokenpkg.TokenHeader, token) diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 0ac77d55a6..071c98ea7e 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -36,7 +36,8 @@ import ( ) const ( - tokenTransportHeader = "X-Reva-Transfer" + // TokenTransportHeader holds the header key for the reva transfer token + TokenTransportHeader = "X-Reva-Transfer" ) func init() { @@ -145,7 +146,7 @@ func (s *svc) doGet(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - token := r.Header.Get(tokenTransportHeader) + token := r.Header.Get(TokenTransportHeader) claims, err := s.verify(ctx, token) if err != nil { err = errors.Wrap(err, "datagateway: error validating transfer token") @@ -188,7 +189,7 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - token := r.Header.Get(tokenTransportHeader) + token := r.Header.Get(TokenTransportHeader) claims, err := s.verify(ctx, token) if err != nil { err = errors.Wrap(err, "datagateway: error validating transfer token") diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 5717500772..8a1f9152d9 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -19,6 +19,7 @@ package ocdav import ( + "github.com/cs3org/reva/internal/http/services/datagateway" "io" "net/http" "path" @@ -103,7 +104,7 @@ func (s *svc) handleGet(w http.ResponseWriter, r *http.Request, ns string) { w.WriteHeader(http.StatusInternalServerError) return } - httpReq.Header.Set("X-Reva-Transfer", dRes.Token) + httpReq.Header.Set(datagateway.TokenTransportHeader, dRes.Token) httpClient := rhttp.GetHTTPClient(ctx) httpRes, err := httpClient.Do(httpReq) From 832aa7de2746707f7371d3023b28f8ad1451c9ab Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 14:56:03 +0200 Subject: [PATCH 2/8] Remove hardcoded data server exposure No-go to have a config variable and then render it useless by hardcoding it to true. --- internal/grpc/services/storageprovider/storageprovider.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index de4653782e..e1860ef888 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -79,9 +79,6 @@ func (c *config) init() { c.DataServerURL = sharedconf.GetDataGateway(c.DataServerURL) - // TODO: Uploads currently don't work when ExposeDataServer is false - c.ExposeDataServer = true - // set sane defaults if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} From 450158fc2e2e2368eec9356fd7bb86f18cc85780 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 14:57:38 +0200 Subject: [PATCH 3/8] Set transfer header on TUS put --- internal/http/services/owncloud/ocdav/put.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 5a9a2be457..23f8e0c25a 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -19,6 +19,7 @@ package ocdav import ( + "github.com/cs3org/reva/internal/http/services/datagateway" "net/http" "path" "regexp" @@ -264,6 +265,7 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { Str("token", tokenpkg.ContextMustGetToken(ctx)). Msg("adding token to header") c.Header.Set(tokenpkg.TokenHeader, tokenpkg.ContextMustGetToken(ctx)) + c.Header.Set(datagateway.TokenTransportHeader, uRes.Token) tusc, err := tus.NewClient(dataServerURL, c) if err != nil { From a89ae6c96d9bd9f5431b0e347b52c04404bcf669 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 15:07:12 +0200 Subject: [PATCH 4/8] If a transfer token exists, append it to upload endpoint Since TUS clients don't understand the reva transfer token, we need to make sure that the upload endpoint comes out as a URL that encodes the full endpoint path, not only the data gateway path. --- internal/http/services/owncloud/ocdav/tus.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index 414f013b7a..c572e1da68 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -21,6 +21,7 @@ package ocdav import ( "net/http" "path" + "strings" "time" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -151,6 +152,15 @@ func (s *svc) handleTusPost(w http.ResponseWriter, r *http.Request, ns string) { return } + // TUS clients don't understand the reva transfer token. We need to append it to the upload endpoint. + // The DataGateway has to take care of pulling it back into the request header upon request arrival. + if uRes.Token != "" { + if !strings.HasSuffix(uRes.UploadEndpoint, "/") { + uRes.UploadEndpoint = uRes.UploadEndpoint + "/" + } + uRes.UploadEndpoint = uRes.UploadEndpoint + uRes.Token + } + w.Header().Set("Location", uRes.UploadEndpoint) // for creation-with-upload extension forward bytes to dataprovider From 01d10a932954f3839a352f2554b42d7d15e1a5fa Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 15:09:27 +0200 Subject: [PATCH 5/8] Code cleanup: sort imports --- internal/http/services/owncloud/ocdav/get.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 8a1f9152d9..69c8d8d117 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -19,13 +19,14 @@ package ocdav import ( - "github.com/cs3org/reva/internal/http/services/datagateway" "io" "net/http" "path" "strconv" "time" + "github.com/cs3org/reva/internal/http/services/datagateway" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/utils" From 9df296ad02ccf62ce66f3b82af5b53d1bfb1d755 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 15:35:28 +0200 Subject: [PATCH 6/8] Grab transfer token from URL if not present in header --- .../http/services/datagateway/datagateway.go | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 071c98ea7e..058e476890 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -23,6 +23,7 @@ import ( "io" "net/http" "net/url" + "path" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" @@ -125,7 +126,14 @@ func addCorsHeader(res http.ResponseWriter) { headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD") } -func (s *svc) verify(ctx context.Context, token string) (*transferClaims, error) { +func (s *svc) verify(ctx context.Context, r *http.Request) (*transferClaims, error) { + // Extract transfer token from request header. If not existing, assume that it's the last path segment instead. + token := r.Header.Get(TokenTransportHeader) + if token == "" { + token = path.Base(r.URL.Path) + r.Header.Set(TokenTransportHeader, token) + } + j, err := jwt.ParseWithClaims(token, &transferClaims{}, func(token *jwt.Token) (interface{}, error) { return []byte(s.conf.TransferSharedSecret), nil }) @@ -146,11 +154,10 @@ func (s *svc) doGet(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - token := r.Header.Get(TokenTransportHeader) - claims, err := s.verify(ctx, token) + claims, err := s.verify(ctx, r) if err != nil { err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Err(err) + log.Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") w.WriteHeader(http.StatusForbidden) return } @@ -189,11 +196,10 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - token := r.Header.Get(TokenTransportHeader) - claims, err := s.verify(ctx, token) + claims, err := s.verify(ctx, r) if err != nil { err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Err(err).Str("token", token).Msg("invalid token") + log.Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") w.WriteHeader(http.StatusForbidden) return } From 2a47958668b3f692928ba0007dec389e31ae62d0 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 21:40:26 +0200 Subject: [PATCH 7/8] Handle PATCH request in data gateway --- .../http/services/datagateway/datagateway.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 058e476890..20a06d1166 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -112,6 +112,9 @@ func (s *svc) setHandler() { case "PUT": s.doPut(w, r) return + case "PATCH": + s.doPatch(w, r) + return default: w.WriteHeader(http.StatusNotImplemented) return @@ -245,3 +248,69 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { log.Err(err).Msg("error writing body after header were set") } } + +// TODO: put and post code is pretty much the same. Should be solved in a nicer way in the long run. +func (s *svc) doPatch(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := appctx.GetLogger(ctx) + + claims, err := s.verify(ctx, r) + if err != nil { + err = errors.Wrap(err, "datagateway: error validating transfer token") + log.Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") + w.WriteHeader(http.StatusForbidden) + return + } + + target := claims.Target + // add query params to target, clients can send checksums and other information. + targetURL, err := url.Parse(target) + if err != nil { + log.Err(err).Msg("datagateway: error parsing target url") + w.WriteHeader(http.StatusInternalServerError) + return + } + + targetURL.RawQuery = r.URL.RawQuery + target = targetURL.String() + + log.Info().Str("target", claims.Target).Msg("sending request to internal data server") + + httpClient := rhttp.GetHTTPClient(ctx) + httpReq, err := rhttp.NewRequest(ctx, "PATCH", target, r.Body) + if err != nil { + log.Err(err).Msg("wrong request") + w.WriteHeader(http.StatusInternalServerError) + return + } + httpReq.Header = r.Header + + httpRes, err := httpClient.Do(httpReq) + if err != nil { + log.Err(err).Msg("error doing PATCH request to data service") + w.WriteHeader(http.StatusInternalServerError) + return + } + + copyHeader(w.Header(), httpRes.Header) + + if httpRes.StatusCode != http.StatusOK { + w.WriteHeader(httpRes.StatusCode) + return + } + + defer httpRes.Body.Close() + w.WriteHeader(http.StatusOK) + _, err = io.Copy(w, httpRes.Body) + if err != nil { + log.Err(err).Msg("error writing body after header were set") + } +} + +func copyHeader(dst, src http.Header) { + for key, values := range src { + for i := range values { + dst.Add(key, values[i]) + } + } +} From 8d0105ce21498fb084409a99b98eb42128f0cf5d Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Wed, 24 Jun 2020 22:42:24 +0200 Subject: [PATCH 8/8] Make linter happy --- internal/http/services/owncloud/ocdav/put.go | 2 +- internal/http/services/owncloud/ocdav/tus.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 23f8e0c25a..c73afbd065 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -19,7 +19,6 @@ package ocdav import ( - "github.com/cs3org/reva/internal/http/services/datagateway" "net/http" "path" "regexp" @@ -29,6 +28,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index c572e1da68..2c322e0f29 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -156,9 +156,9 @@ func (s *svc) handleTusPost(w http.ResponseWriter, r *http.Request, ns string) { // The DataGateway has to take care of pulling it back into the request header upon request arrival. if uRes.Token != "" { if !strings.HasSuffix(uRes.UploadEndpoint, "/") { - uRes.UploadEndpoint = uRes.UploadEndpoint + "/" + uRes.UploadEndpoint += "/" } - uRes.UploadEndpoint = uRes.UploadEndpoint + uRes.Token + uRes.UploadEndpoint += uRes.Token } w.Header().Set("Location", uRes.UploadEndpoint)