upload directly to dataprovider (#4065)

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

changelog/unreleased/upload-directly-to-dataprovider.md

@@ -0,0 +1,6 @@
+Enhancement: upload directly to dataprovider
+
+The ocdav service can now bypass the datagateway if it is configured with a transfer secret. This prevents unnecessary roundtrips and halves the network traffic during uploads for the proxy.
+
+https://github.com/cs3org/reva/pull/4065
+https://github.com/owncloud/ocis/issues/6296

internal/http/services/datagateway/datagateway.go

@@ -58,8 +58,8 @@ func init() {
 	global.Register("datagateway", New)
 }
 
-// transferClaims are custom claims for a JWT token to be used between the metadata and data gateways.
-type transferClaims struct {
+// TransferClaims are custom claims for a JWT token to be used between the metadata and data gateways.
+type TransferClaims struct {
 	jwt.StandardClaims
 	Target string `json:"target"`
 }
@@ -161,30 +161,34 @@ func addCorsHeader(res http.ResponseWriter) {
 	headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD")
 }
 
-func (s *svc) verify(ctx context.Context, r *http.Request) (*transferClaims, error) {
-	// Extract transfer token from request header. If not existing, assume that it's the last path segment instead.
-	token := r.Header.Get(TokenTransportHeader)
-	if token == "" {
-		token = path.Base(r.URL.Path)
-		r.Header.Set(TokenTransportHeader, token)
-	}
-
-	j, err := jwt.ParseWithClaims(token, &transferClaims{}, func(token *jwt.Token) (interface{}, error) {
-		return []byte(s.conf.TransferSharedSecret), nil
+// Verify a transfer token against the given secret
+func Verify(ctx context.Context, token string, secret string) (*TransferClaims, error) {
+	j, err := jwt.ParseWithClaims(token, &TransferClaims{}, func(token *jwt.Token) (interface{}, error) {
+		return []byte(secret), nil
 	})
 
 	if err != nil {
 		return nil, errors.Wrap(err, "error parsing token")
 	}
 
-	if claims, ok := j.Claims.(*transferClaims); ok && j.Valid {
+	if claims, ok := j.Claims.(*TransferClaims); ok && j.Valid {
 		return claims, nil
 	}
-
 	err = errtypes.InvalidCredentials("token invalid")
 	return nil, err
 }
 
+func (s *svc) verify(ctx context.Context, r *http.Request) (*TransferClaims, error) {
+	// Extract transfer token from request header. If not existing, assume that it's the last path segment instead.
+	token := r.Header.Get(TokenTransportHeader)
+	if token == "" {
+		token = path.Base(r.URL.Path)
+		r.Header.Set(TokenTransportHeader, token)
+	}
+
+	return Verify(ctx, token, s.conf.TransferSharedSecret)
+}
+
 func (s *svc) doHead(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	log := appctx.GetLogger(ctx)

internal/http/services/owncloud/ocdav/ocdav.go

@@ -85,6 +85,8 @@ type Config struct {
 	Product                string                            `mapstructure:"product"`
 	ProductName            string                            `mapstructure:"product_name"`
 	ProductVersion         string                            `mapstructure:"product_version"`
+	// optional, if set will unpack the transfer token and directly send uploads to the data provider
+	TransferSharedSecret string `mapstructure:"transfer_shared_secret"`
 
 	NameValidation NameValidation `mapstructure:"validation"`
 

internal/http/services/owncloud/ocdav/put.go

@@ -298,6 +298,18 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		}
 	}
 
+	// if we know the transfer secret we can directly talk to the dataprovider
+	if s.c.TransferSharedSecret != "" {
+		claims, err := datagateway.Verify(ctx, token, s.c.TransferSharedSecret)
+		if err != nil {
+			log.Error().Err(err).Msg("error verifying transfer token")
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		// directly send request to target
+		ep = claims.Target
+	}
+
 	httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, r.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)

internal/http/services/owncloud/ocdav/tus.go

@@ -32,6 +32,7 @@ import (
 	link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
+	"github.com/cs3org/reva/v2/internal/http/services/datagateway"
 	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/errors"
 	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net"
 	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/spacelookup"
@@ -254,6 +255,18 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http.
 		}
 		var httpRes *http.Response
 
+		// if we know the transfer secret we can directly talk to the dataprovider
+		if s.c.TransferSharedSecret != "" {
+			claims, err := datagateway.Verify(ctx, token, s.c.TransferSharedSecret)
+			if err != nil {
+				log.Error().Err(err).Msg("error verifying transfer token")
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			// directly send request to target
+			ep = claims.Target
+		}
+
 		httpReq, err := rhttp.NewRequest(ctx, http.MethodPatch, ep, r.Body)
 		if err != nil {
 			log.Debug().Err(err).Msg("wrong request")

pkg/micro/ocdav/option.go

@@ -43,8 +43,9 @@ type Options struct {
 	Context   context.Context
 	// Metrics   *metrics.Metrics
 	// Flags     []cli.Flag
-	Name      string
-	JWTSecret string
+	Name           string
+	JWTSecret      string
+	TransferSecret string
 
 	FavoriteManager favorite.Manager
 	GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
@@ -109,6 +110,13 @@ func JWTSecret(s string) Option {
 	}
 }
 
+// TransferSecret provides a function to set the transfer secret option.
+func TransferSecret(s string) Option {
+	return func(o *Options) {
+		o.config.TransferSharedSecret = s
+	}
+}
+
 // MachineAuthAPIKey provides a function to set the machine auth api key option.
 func MachineAuthAPIKey(s string) Option {
 	return func(o *Options) {