From 283ecff63fe42a119103020f4934838228f8f0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 14 Jul 2023 16:02:26 +0200 Subject: [PATCH] upload directly to dataprovider (#4065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../upload-directly-to-dataprovider.md | 6 ++++ .../http/services/datagateway/datagateway.go | 32 +++++++++++-------- .../http/services/owncloud/ocdav/ocdav.go | 2 ++ internal/http/services/owncloud/ocdav/put.go | 12 +++++++ internal/http/services/owncloud/ocdav/tus.go | 13 ++++++++ pkg/micro/ocdav/option.go | 12 +++++-- 6 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 changelog/unreleased/upload-directly-to-dataprovider.md diff --git a/changelog/unreleased/upload-directly-to-dataprovider.md b/changelog/unreleased/upload-directly-to-dataprovider.md new file mode 100644 index 0000000000..586d7ea49b --- /dev/null +++ b/changelog/unreleased/upload-directly-to-dataprovider.md @@ -0,0 +1,6 @@ +Enhancement: upload directly to dataprovider + +The ocdav service can now bypass the datagateway if it is configured with a transfer secret. This prevents unnecessary roundtrips and halves the network traffic during uploads for the proxy. + +https://github.com/cs3org/reva/pull/4065 +https://github.com/owncloud/ocis/issues/6296 \ No newline at end of file diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 69fe36f5ea..5e66ad413d 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -58,8 +58,8 @@ func init() { global.Register("datagateway", New) } -// transferClaims are custom claims for a JWT token to be used between the metadata and data gateways. -type transferClaims struct { +// TransferClaims are custom claims for a JWT token to be used between the metadata and data gateways. +type TransferClaims struct { jwt.StandardClaims Target string `json:"target"` } @@ -161,30 +161,34 @@ func addCorsHeader(res http.ResponseWriter) { headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD") } -func (s *svc) verify(ctx context.Context, r *http.Request) (*transferClaims, error) { - // Extract transfer token from request header. If not existing, assume that it's the last path segment instead. - token := r.Header.Get(TokenTransportHeader) - if token == "" { - token = path.Base(r.URL.Path) - r.Header.Set(TokenTransportHeader, token) - } - - j, err := jwt.ParseWithClaims(token, &transferClaims{}, func(token *jwt.Token) (interface{}, error) { - return []byte(s.conf.TransferSharedSecret), nil +// Verify a transfer token against the given secret +func Verify(ctx context.Context, token string, secret string) (*TransferClaims, error) { + j, err := jwt.ParseWithClaims(token, &TransferClaims{}, func(token *jwt.Token) (interface{}, error) { + return []byte(secret), nil }) if err != nil { return nil, errors.Wrap(err, "error parsing token") } - if claims, ok := j.Claims.(*transferClaims); ok && j.Valid { + if claims, ok := j.Claims.(*TransferClaims); ok && j.Valid { return claims, nil } - err = errtypes.InvalidCredentials("token invalid") return nil, err } +func (s *svc) verify(ctx context.Context, r *http.Request) (*TransferClaims, error) { + // Extract transfer token from request header. If not existing, assume that it's the last path segment instead. + token := r.Header.Get(TokenTransportHeader) + if token == "" { + token = path.Base(r.URL.Path) + r.Header.Set(TokenTransportHeader, token) + } + + return Verify(ctx, token, s.conf.TransferSharedSecret) +} + func (s *svc) doHead(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index e0291a3194..e96e1b445f 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -85,6 +85,8 @@ type Config struct { Product string `mapstructure:"product"` ProductName string `mapstructure:"product_name"` ProductVersion string `mapstructure:"product_version"` + // optional, if set will unpack the transfer token and directly send uploads to the data provider + TransferSharedSecret string `mapstructure:"transfer_shared_secret"` NameValidation NameValidation `mapstructure:"validation"` diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 58063e5ee2..e6e8322f7b 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -298,6 +298,18 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ } } + // if we know the transfer secret we can directly talk to the dataprovider + if s.c.TransferSharedSecret != "" { + claims, err := datagateway.Verify(ctx, token, s.c.TransferSharedSecret) + if err != nil { + log.Error().Err(err).Msg("error verifying transfer token") + w.WriteHeader(http.StatusInternalServerError) + return + } + // directly send request to target + ep = claims.Target + } + httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, r.Body) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index 5371472f71..0d4e30fa7c 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -32,6 +32,7 @@ import ( link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/internal/http/services/datagateway" "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/errors" "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spacelookup" @@ -254,6 +255,18 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http. } var httpRes *http.Response + // if we know the transfer secret we can directly talk to the dataprovider + if s.c.TransferSharedSecret != "" { + claims, err := datagateway.Verify(ctx, token, s.c.TransferSharedSecret) + if err != nil { + log.Error().Err(err).Msg("error verifying transfer token") + w.WriteHeader(http.StatusInternalServerError) + return + } + // directly send request to target + ep = claims.Target + } + httpReq, err := rhttp.NewRequest(ctx, http.MethodPatch, ep, r.Body) if err != nil { log.Debug().Err(err).Msg("wrong request") diff --git a/pkg/micro/ocdav/option.go b/pkg/micro/ocdav/option.go index a6ebba9429..00da05f59b 100644 --- a/pkg/micro/ocdav/option.go +++ b/pkg/micro/ocdav/option.go @@ -43,8 +43,9 @@ type Options struct { Context context.Context // Metrics *metrics.Metrics // Flags []cli.Flag - Name string - JWTSecret string + Name string + JWTSecret string + TransferSecret string FavoriteManager favorite.Manager GatewaySelector pool.Selectable[gateway.GatewayAPIClient] @@ -109,6 +110,13 @@ func JWTSecret(s string) Option { } } +// TransferSecret provides a function to set the transfer secret option. +func TransferSecret(s string) Option { + return func(o *Options) { + o.config.TransferSharedSecret = s + } +} + // MachineAuthAPIKey provides a function to set the machine auth api key option. func MachineAuthAPIKey(s string) Option { return func(o *Options) {