diff --git a/go.mod b/go.mod index ebaf458c70f..07029ffa090 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.18.1-0.20240208163049-039a779e377b + github.com/cs3org/reva/v2 v2.18.1-0.20240220100621-283b39bc20e6 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e diff --git a/go.sum b/go.sum index d46b5da017f..5e96e7eaaea 100644 --- a/go.sum +++ b/go.sum @@ -1019,8 +1019,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.18.1-0.20240208163049-039a779e377b h1:nTKOX+nyoljL7CDnfEjec7nzUKc1+7Jc89Q/uRRc2Zs= -github.com/cs3org/reva/v2 v2.18.1-0.20240208163049-039a779e377b/go.mod h1:GRUrOp5HbFVwZTgR9bVrMZ/MvVy+Jhxw1PdMmhhKP9E= +github.com/cs3org/reva/v2 v2.18.1-0.20240220100621-283b39bc20e6 h1:9O1K3ZaeK5752WYKqRH+ESnS/zIEZpl+ss6FsYnpUmc= +github.com/cs3org/reva/v2 v2.18.1-0.20240220100621-283b39bc20e6/go.mod h1:GRUrOp5HbFVwZTgR9bVrMZ/MvVy+Jhxw1PdMmhhKP9E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/services/sharing/pkg/config/config.go b/services/sharing/pkg/config/config.go index f537400349a..1e3a7fb33e1 100644 --- a/services/sharing/pkg/config/config.go +++ b/services/sharing/pkg/config/config.go @@ -21,6 +21,8 @@ type Config struct { SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"SHARING_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` + EnableResharing bool `yaml:"enable_resharing" env:"OCIS_ENABLE_RESHARING;SHARING_ENABLE_RESHARING" desc:"Changing this value is NOT supported. Enables the support for resharing."` + UserSharingDriver string `yaml:"user_sharing_driver" env:"SHARING_USER_DRIVER" desc:"Driver to be used to persist shares. Supported values are 'jsoncs3', 'json', 'cs3' (deprecated) and 'owncloudsql'."` UserSharingDrivers UserSharingDrivers `yaml:"user_sharing_drivers"` PublicSharingDriver string `yaml:"public_sharing_driver" env:"SHARING_PUBLIC_DRIVER" desc:"Driver to be used to persist public shares. Supported values are 'jsoncs3', 'json' and 'cs3' (deprecated)."` diff --git a/services/sharing/pkg/config/defaults/defaultconfig.go b/services/sharing/pkg/config/defaults/defaultconfig.go index 57026e0b9a3..20d86a739a4 100644 --- a/services/sharing/pkg/config/defaults/defaultconfig.go +++ b/services/sharing/pkg/config/defaults/defaultconfig.go @@ -35,6 +35,7 @@ func DefaultConfig() *config.Config { Name: "sharing", }, Reva: shared.DefaultRevaConfig(), + EnableResharing: true, UserSharingDriver: "jsoncs3", UserSharingDrivers: config.UserSharingDrivers{ JSON: config.UserSharingJSONDriver{ diff --git a/services/sharing/pkg/revaconfig/config.go b/services/sharing/pkg/revaconfig/config.go index 8f02b796de4..d91879eed7f 100644 --- a/services/sharing/pkg/revaconfig/config.go +++ b/services/sharing/pkg/revaconfig/config.go @@ -86,6 +86,7 @@ func SharingConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string] }, }, }, + "disable_resharing": cfg.EnableResharing, }, "publicshareprovider": map[string]interface{}{ "gateway_addr": cfg.Reva.Address, diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 43cdaa61d8e..0697ee7a584 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -84,6 +84,17 @@ type HTTPConfig struct { Namespace string `yaml:"-"` Protocol string `yaml:"protocol" env:"STORAGE_USERS_HTTP_PROTOCOL" desc:"The transport protocol of the HTTP service."` Prefix string + CORS CORS `yaml:"cors"` +} + +// CORS defines the available cors configuration. +type CORS struct { + AllowedOrigins []string `yaml:"allow_origins" env:"OCIS_CORS_ALLOW_ORIGINS;STORAGE_USERS_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details."` + AllowedMethods []string `yaml:"allow_methods" env:"OCIS_CORS_ALLOW_METHODS;STORAGE_USERS_CORS_ALLOW_METHODS" desc:"A list of allowed CORS methods. See following chapter for more details: *Access-Control-Request-Method* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Method. See the Environment Variable Types description for more details."` + AllowedHeaders []string `yaml:"allow_headers" env:"OCIS_CORS_ALLOW_HEADERS;STORAGE_USERS_CORS_ALLOW_HEADERS" desc:"A list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers. See the Environment Variable Types description for more details."` + AllowCredentials bool `yaml:"allow_credentials" env:"OCIS_CORS_ALLOW_CREDENTIALS;STORAGE_USERS_CORS_ALLOW_CREDENTIALS" desc:"Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials."` + ExposedHeaders []string `yaml:"expose_headers" env:"OCIS_CORS_EXPOSE_HEADERS;STORAGE_USERS_CORS_EXPOSE_HEADERS" desc:"A list of exposed CORS headers. See following chapter for more details: *Access-Control-Expose-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers. See the Environment Variable Types description for more details."` + MaxAge uint `yaml:"max_age" env:"OCIS_CORS_MAX_AGE;STORAGE_USERS_CORS_MAX_AGE" desc:"The max cache duration of preflight headers. See following chapter for more details: *Access-Control-Max-Age* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Max-Age. See the Environment Variable Types description for more details."` } // Drivers combine all storage driver configurations @@ -132,14 +143,20 @@ type S3NGDriver struct { Propagator string `yaml:"propagator" env:"OCIS_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_S3NG_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option."` AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"` // Root is the absolute path to the location of the data - Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."` - UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."` - PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."` - Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."` - AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."` - SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."` - Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."` - Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."` + Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/users."` + UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory."` + PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'."` + Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket."` + AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket."` + SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket."` + Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket."` + Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket."` + DisableContentSha256 bool `yaml:"put_object_disable_content_sha254" env:"STORAGE_USERS_S3NG_PUT_OBJECT_DISABLE_CONTENT_SHA256" desc:"Disable sending content sha256 when copying objects to S3."` + DisableMultipart bool `yaml:"put_object_disable_multipart" env:"STORAGE_USERS_S3NG_PUT_OBJECT_DISABLE_MULTIPART" desc:"Disable multipart uploads when copying objects to S3"` + SendContentMd5 bool `yaml:"put_object_send_content_md5" env:"STORAGE_USERS_S3NG_PUT_OBJECT_SEND_CONTENT_MD5" desc:"Send a Content-MD5 header when copying objects to S3."` + ConcurrentStreamParts bool `yaml:"put_object_concurrent_stream_parts" env:"STORAGE_USERS_S3NG_PUT_OBJECT_CONCURRENT_STREAM_PARTS" desc:"Always precreate parts when copying objects to S3."` + NumThreads uint `yaml:"put_object_num_threads" env:"STORAGE_USERS_S3NG_PUT_OBJECT_NUM_THREADS" desc:"Number of concurrent uploads to use when copying objects to S3."` + PartSize uint64 `yaml:"put_object_part_size" env:"STORAGE_USERS_S3NG_PUT_OBJECT_PART_SIZE" desc:"Part size for concurrent uploads to S3."` // PersonalSpaceAliasTemplate contains the template used to construct // the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"` PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_S3NG_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases."` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 65510c8ec0e..b615db8ba7a 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -37,6 +37,49 @@ func DefaultConfig() *config.Config { Namespace: "com.owncloud.web", Protocol: "tcp", Prefix: "data", + CORS: config.CORS{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{ + "POST", + "HEAD", + "PATCH", + "OPTIONS", + "GET", + "DELETE", + }, + AllowedHeaders: []string{ + "Authorization", + "Origin", + "X-Requested-With", + "X-Request-Id", + "X-HTTP-Method-Override", + "Content-Type", + "Upload-Length", + "Upload-Offset", + "Tus-Resumable", + "Upload-Metadata", + "Upload-Defer-Length", + "Upload-Concat", + "Upload-Incomplete", + "Upload-Draft-Interop-Version", + }, + AllowCredentials: true, + ExposedHeaders: []string{ + "Upload-Offset", + "Location", + "Upload-Length", + "Tus-Version", + "Tus-Resumable", + "Tus-Max-Size", + "Tus-Extension", + "Upload-Metadata", + "Upload-Defer-Length", + "Upload-Concat", + "Upload-Incomplete", + "Upload-Draft-Interop-Version", + }, + MaxAge: 86400, + }, }, Service: config.Service{ Name: "storage-users", @@ -69,6 +112,9 @@ func DefaultConfig() *config.Config { ShareFolder: "/Shares", UserLayout: "{{.Id.OpaqueId}}", Region: "default", + SendContentMd5: true, + ConcurrentStreamParts: true, + NumThreads: 4, PersonalSpaceAliasTemplate: "{{.SpaceType}}/{{.User.Username | lower}}", GeneralSpaceAliasTemplate: "{{.SpaceType}}/{{.SpaceName | replace \" \" \"-\" | lower}}", PermissionsEndpoint: "com.owncloud.api.settings", diff --git a/services/storage-users/pkg/revaconfig/config.go b/services/storage-users/pkg/revaconfig/config.go index f8ecffd9b73..aaa31f59e98 100644 --- a/services/storage-users/pkg/revaconfig/config.go +++ b/services/storage-users/pkg/revaconfig/config.go @@ -2,6 +2,9 @@ package revaconfig import ( + "strconv" + "strings" + "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" ) @@ -74,6 +77,31 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { "nats_enable_tls": cfg.Events.EnableTLS, "nats_username": cfg.Events.AuthUsername, "nats_password": cfg.Events.AuthPassword, + "data_txs": map[string]interface{}{ + "simple": map[string]interface{}{ + "cache_store": "noop", + "cache_database": "system", + "cache_table": "stat", + }, + "spaces": map[string]interface{}{ + "cache_store": "noop", + "cache_database": "system", + "cache_table": "stat", + }, + "tus": map[string]interface{}{ + "cache_store": "noop", + "cache_database": "system", + "cache_table": "stat", + "cors_enabled": true, + // allow_origin is configured as a regex in tusd, so we concatenate the configured values into a regex + "cors_allow_origin": "(" + strings.ReplaceAll(strings.Join(cfg.HTTP.CORS.AllowedOrigins, "|"), "*", ".*") + ")", + "cors_allow_credentials": cfg.HTTP.CORS.AllowCredentials, + "cors_allow_methods": strings.Join(cfg.HTTP.CORS.AllowedMethods, ","), + "cors_allow_headers": strings.Join(cfg.HTTP.CORS.AllowedHeaders, ","), + "cors_max_age": strconv.FormatUint(uint64(cfg.HTTP.CORS.MaxAge), 10), + "cors_expose_headers": strings.Join(cfg.HTTP.CORS.ExposedHeaders, ","), + }, + }, }, }, }, diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 0a44caf74c8..841a3190998 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -243,6 +243,12 @@ func S3NG(cfg *config.Config) map[string]interface{} { "s3.secret_key": cfg.Drivers.S3NG.SecretKey, "s3.endpoint": cfg.Drivers.S3NG.Endpoint, "s3.bucket": cfg.Drivers.S3NG.Bucket, + "s3.disable_content_sha254": cfg.Drivers.S3NG.DisableContentSha256, + "s3.disable_multipart": cfg.Drivers.S3NG.DisableMultipart, + "s3.send_content_md5": cfg.Drivers.S3NG.SendContentMd5, + "s3.concurrent_stream_parts": cfg.Drivers.S3NG.ConcurrentStreamParts, + "s3.num_threads": cfg.Drivers.S3NG.NumThreads, + "s3.part_size": cfg.Drivers.S3NG.PartSize, "max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go index 0710d007864..0c503e7cc29 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go @@ -55,6 +55,7 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` GatewayAddr string `mapstructure:"gateway_addr"` AllowedPathsForShares []string `mapstructure:"allowed_paths_for_shares"` + DisableResharing bool `mapstructure:"disable_resharing"` } func (c *config) init() { @@ -67,6 +68,7 @@ type service struct { sm share.Manager gatewaySelector pool.Selectable[gateway.GatewayAPIClient] allowedPathsForShares []*regexp.Regexp + disableResharing bool } func getShareManager(c *config) (share.Manager, error) { @@ -127,15 +129,16 @@ func NewDefault(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error return nil, err } - return New(gatewaySelector, sm, allowedPathsForShares), nil + return New(gatewaySelector, sm, allowedPathsForShares, c.DisableResharing), nil } // New creates a new user share provider svc -func New(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], sm share.Manager, allowedPathsForShares []*regexp.Regexp) rgrpc.Service { +func New(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], sm share.Manager, allowedPathsForShares []*regexp.Regexp, disableResharing bool) rgrpc.Service { service := &service{ sm: sm, gatewaySelector: gatewaySelector, allowedPathsForShares: allowedPathsForShares, + disableResharing: disableResharing, } return service @@ -157,6 +160,13 @@ func (s *service) CreateShare(ctx context.Context, req *collaboration.CreateShar log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) + // when resharing is disabled grants must not allow grant permissions + if s.disableResharing && HasGrantPermissions(req.GetGrant().GetPermissions().GetPermissions()) { + return &collaboration.CreateShareResponse{ + Status: status.NewInvalidArg(ctx, "resharing not supported"), + }, nil + } + gatewayClient, err := s.gatewaySelector.Next() if err != nil { return nil, err @@ -235,6 +245,10 @@ func (s *service) CreateShare(ctx context.Context, req *collaboration.CreateShar }, nil } +func HasGrantPermissions(p *provider.ResourcePermissions) bool { + return p.GetAddGrant() || p.GetUpdateGrant() || p.GetRemoveGrant() || p.GetDenyGrant() +} + func (s *service) RemoveShare(ctx context.Context, req *collaboration.RemoveShareRequest) (*collaboration.RemoveShareResponse, error) { log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) @@ -327,6 +341,14 @@ func (s *service) ListShares(ctx context.Context, req *collaboration.ListSharesR func (s *service) UpdateShare(ctx context.Context, req *collaboration.UpdateShareRequest) (*collaboration.UpdateShareResponse, error) { log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) + + // when resharing is disabled grants must not allow grant permissions + if s.disableResharing && HasGrantPermissions(req.GetShare().GetPermissions().GetPermissions()) { + return &collaboration.UpdateShareResponse{ + Status: status.NewInvalidArg(ctx, "resharing not supported"), + }, nil + } + gatewayClient, err := s.gatewaySelector.Next() if err != nil { return nil, err diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/datagateway/datagateway.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/datagateway/datagateway.go index 13411fc36dc..53ba79af817 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/datagateway/datagateway.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/datagateway/datagateway.go @@ -50,8 +50,6 @@ func init() { const ( // TokenTransportHeader holds the header key for the reva transfer token TokenTransportHeader = "X-Reva-Transfer" - // UploadExpiresHeader holds the timestamp for the transport token expiry, defined in https://tus.io/protocols/resumable-upload.html#expiration - UploadExpiresHeader = "Upload-Expires" ) func init() { @@ -133,31 +131,13 @@ func (s *svc) setHandler() { semconv.HTTPURLKey.String(r.URL.String()), ) r = r.WithContext(ctx) - switch r.Method { - case "HEAD": - s.doHead(w, r) - return - case "GET": - s.doGet(w, r) - return - case "PUT": - s.doPut(w, r) - return - case "PATCH": - s.doPatch(w, r) - return - case "OPTIONS": - s.doOptions(w, r) - return - default: - w.WriteHeader(http.StatusNotImplemented) - return - } + s.doRequest(w, r) }) } +// verify extracts the transfer token from the request +// If it is not set as header we assume that it's the last path segment instead. func (s *svc) verify(ctx context.Context, r *http.Request) (*transferClaims, error) { - // Extract transfer token from request header. If not existing, assume that it's the last path segment instead. token := r.Header.Get(TokenTransportHeader) if token == "" { token = path.Base(r.URL.Path) @@ -180,112 +160,7 @@ func (s *svc) verify(ctx context.Context, r *http.Request) (*transferClaims, err return nil, err } -func (s *svc) doHead(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := appctx.GetLogger(ctx) - - claims, err := s.verify(ctx, r) - if err != nil { - err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Error().Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") - w.WriteHeader(http.StatusForbidden) - return - } - - log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") - - httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, "HEAD", claims.Target, nil) - if err != nil { - log.Error().Err(err).Msg("wrong request") - w.WriteHeader(http.StatusInternalServerError) - return - } - httpReq.Header = r.Header - - httpRes, err := httpClient.Do(httpReq) - if err != nil { - log.Error().Err(err).Msg("error doing HEAD request to data service") - w.WriteHeader(http.StatusInternalServerError) - return - } - defer httpRes.Body.Close() - - copyHeader(w.Header(), httpRes.Header) - - // add upload expiry / transfer token expiry header for tus https://tus.io/protocols/resumable-upload.html#expiration - w.Header().Set(UploadExpiresHeader, time.Unix(claims.ExpiresAt, 0).Format(time.RFC1123)) - - if httpRes.StatusCode != http.StatusOK { - // swallow the body and set content-length to 0 to prevent reverse proxies from trying to read from it - w.Header().Set("Content-Length", "0") - w.WriteHeader(httpRes.StatusCode) - return - } - - w.WriteHeader(http.StatusOK) -} - -func (s *svc) doGet(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := appctx.GetLogger(ctx) - - claims, err := s.verify(ctx, r) - if err != nil { - err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Error().Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") - w.WriteHeader(http.StatusForbidden) - return - } - - log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") - - httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, "GET", claims.Target, nil) - if err != nil { - log.Error().Err(err).Msg("wrong request") - w.WriteHeader(http.StatusInternalServerError) - return - } - httpReq.Header = r.Header - - httpRes, err := httpClient.Do(httpReq) - if err != nil { - log.Error().Err(err).Msg("error doing GET request to data service") - w.WriteHeader(http.StatusInternalServerError) - return - } - defer httpRes.Body.Close() - - copyHeader(w.Header(), httpRes.Header) - switch httpRes.StatusCode { - case http.StatusOK: - case http.StatusPartialContent: - default: - // swallow the body and set content-length to 0 to prevent reverse proxies from trying to read from it - w.Header().Set("Content-Length", "0") - w.WriteHeader(httpRes.StatusCode) - return - } - w.WriteHeader(httpRes.StatusCode) - - var c int64 - c, err = io.Copy(w, httpRes.Body) - if err != nil { - log.Error().Err(err).Msg("error writing body after headers were sent") - } - if httpRes.Header.Get("Content-Length") != "" { - i, err := strconv.ParseInt(httpRes.Header.Get("Content-Length"), 10, 64) - if err != nil { - log.Error().Err(err).Str("content-length", httpRes.Header.Get("Content-Length")).Msg("invalid content length in dataprovider response") - } - if i != c { - log.Error().Int64("content-length", i).Int64("transferred-bytes", c).Msg("content length vs transferred bytes mismatch") - } - } -} - -func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { +func (s *svc) doRequest(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) @@ -309,10 +184,9 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { targetURL.RawQuery = r.URL.RawQuery target = targetURL.String() - log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") + log.Debug().Str("target", target).Msg("sending request to internal data server") - httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, "PUT", target, r.Body) + httpReq, err := rhttp.NewRequest(ctx, r.Method, target, r.Body) if err != nil { log.Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) @@ -321,68 +195,9 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { httpReq.Header = r.Header httpReq.ContentLength = r.ContentLength - httpRes, err := httpClient.Do(httpReq) + httpRes, err := s.client.Do(httpReq) if err != nil { - log.Err(err).Msg("error doing PUT request to data service") - w.WriteHeader(http.StatusInternalServerError) - return - } - defer httpRes.Body.Close() - - copyHeader(w.Header(), httpRes.Header) - if httpRes.StatusCode != http.StatusOK { - // swallow the body and set content-length to 0 to prevent reverse proxies from trying to read from it - w.Header().Set("Content-Length", "0") - w.WriteHeader(httpRes.StatusCode) - return - } - - w.WriteHeader(http.StatusOK) - _, err = io.Copy(w, httpRes.Body) - if err != nil { - log.Err(err).Msg("error writing body after header were set") - } -} - -// TODO: put and post code is pretty much the same. Should be solved in a nicer way in the long run. -func (s *svc) doPatch(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := appctx.GetLogger(ctx) - - claims, err := s.verify(ctx, r) - if err != nil { - err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") - w.WriteHeader(http.StatusForbidden) - return - } - - target := claims.Target - // add query params to target, clients can send checksums and other information. - targetURL, err := url.Parse(target) - if err != nil { - log.Err(err).Msg("datagateway: error parsing target url") - w.WriteHeader(http.StatusInternalServerError) - return - } - - targetURL.RawQuery = r.URL.RawQuery - target = targetURL.String() - - log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") - - httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, "PATCH", target, r.Body) - if err != nil { - log.Err(err).Msg("wrong request") - w.WriteHeader(http.StatusInternalServerError) - return - } - httpReq.Header = r.Header - - httpRes, err := httpClient.Do(httpReq) - if err != nil { - log.Err(err).Msg("error doing PATCH request to data service") + log.Err(err).Msg("error doing " + r.Method + " request to data service") w.WriteHeader(http.StatusInternalServerError) return } @@ -395,58 +210,22 @@ func (s *svc) doPatch(w http.ResponseWriter, r *http.Request) { w.WriteHeader(httpRes.StatusCode) return } - w.WriteHeader(httpRes.StatusCode) - _, err = io.Copy(w, httpRes.Body) - if err != nil { - log.Err(err).Msg("error writing body after header were set") - } -} - -func (s *svc) doOptions(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := appctx.GetLogger(ctx) - - claims, err := s.verify(ctx, r) - if err != nil { - err = errors.Wrap(err, "datagateway: error validating transfer token") - log.Error().Err(err).Str("token", r.Header.Get(TokenTransportHeader)).Msg("invalid transfer token") - w.WriteHeader(http.StatusForbidden) - return - } - - log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") - httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, "OPTIONS", claims.Target, nil) - if err != nil { - log.Error().Err(err).Msg("wrong request") - w.WriteHeader(http.StatusInternalServerError) - return - } - httpReq.Header = r.Header - - httpRes, err := httpClient.Do(httpReq) + var c int64 + c, err = io.Copy(w, httpRes.Body) if err != nil { - log.Error().Err(err).Msg("error doing OPTIONS request to data service") - w.WriteHeader(http.StatusInternalServerError) - return + log.Err(err).Msg("error writing body after header were set") } - defer httpRes.Body.Close() - - copyHeader(w.Header(), httpRes.Header) - - // add upload expiry / transfer token expiry header for tus https://tus.io/protocols/resumable-upload.html#expiration - w.Header().Set(UploadExpiresHeader, time.Unix(claims.ExpiresAt, 0).Format(time.RFC1123)) - - if httpRes.StatusCode != http.StatusOK { - // swallow the body and set content-length to 0 to prevent reverse proxies from trying to read from it - w.Header().Set("Content-Length", "0") - w.WriteHeader(httpRes.StatusCode) - return + if httpRes.Header.Get("Content-Length") != "" { + i, err := strconv.ParseInt(httpRes.Header.Get("Content-Length"), 10, 64) + if err != nil { + log.Error().Err(err).Str("content-length", httpRes.Header.Get("Content-Length")).Msg("invalid content length in dataprovider response") + } + if i != c { + log.Error().Int64("content-length", i).Int64("transferred-bytes", c).Msg("content length vs transferred bytes mismatch") + } } - - w.WriteHeader(http.StatusOK) } func copyHeader(dst, src http.Header) { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go index 9c744e75401..48a40b764f0 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go @@ -37,11 +37,22 @@ import ( type Blobstore struct { client *minio.Client + defaultPutOptions Options + bucket string } +type Options struct { + DisableContentSha256 bool + DisableMultipart bool + SendContentMd5 bool + ConcurrentStreamParts bool + NumThreads uint + PartSize uint64 +} + // New returns a new Blobstore -func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, error) { +func New(endpoint, region, bucket, accessKey, secretKey string, defaultPutOptions Options) (*Blobstore, error) { u, err := url.Parse(endpoint) if err != nil { return nil, errors.Wrap(err, "failed to parse s3 endpoint") @@ -58,8 +69,9 @@ func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, err } return &Blobstore{ - client: client, - bucket: bucket, + client: client, + bucket: bucket, + defaultPutOptions: defaultPutOptions, }, nil } @@ -71,7 +83,15 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { } defer reader.Close() - _, err = bs.client.PutObject(context.Background(), bs.bucket, bs.path(node), reader, node.Blobsize, minio.PutObjectOptions{ContentType: "application/octet-stream", SendContentMd5: true}) + _, err = bs.client.PutObject(context.Background(), bs.bucket, bs.path(node), reader, node.Blobsize, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: bs.defaultPutOptions.SendContentMd5, + ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts, + NumThreads: bs.defaultPutOptions.NumThreads, + PartSize: bs.defaultPutOptions.PartSize, + DisableMultipart: bs.defaultPutOptions.DisableMultipart, + DisableContentSha256: bs.defaultPutOptions.DisableContentSha256, + }) if err != nil { return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.path(node), bs.bucket) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go index 877a7d71891..54c038af79a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/option.go @@ -43,6 +43,24 @@ type Options struct { // Secret key for the s3 blobstore S3SecretKey string `mapstructure:"s3.secret_key"` + + // disable sending content sha256 + DisableContentSha256 bool `mapstructure:"s3.disable_content_sha254"` + + // disable multipart uploads + DisableMultipart bool `mapstructure:"s3.disable_multipart"` + + // enable sending content md5, defaults to true if unset + SendContentMd5 bool `mapstructure:"s3.send_content_md5"` + + // use concurrent stream parts + ConcurrentStreamParts bool `mapstructure:"s3.concurrent_stream_parts"` + + // number of concurrent uploads + NumThreads uint `mapstructure:"s3.num_threads"` + + // part size for concurrent uploads + PartSize uint64 `mapstructure:"s3.part_size"` } // S3ConfigComplete return true if all required s3 fields are set @@ -60,5 +78,16 @@ func parseConfig(m map[string]interface{}) (*Options, error) { err = errors.Wrap(err, "error decoding conf") return nil, err } + + // if unset we set these defaults + if m["s3.send_content_md5"] == nil { + o.SendContentMd5 = true + } + if m["s3.concurrent_stream_parts"] == nil { + o.ConcurrentStreamParts = true + } + if m["s3.num_threads"] == nil { + o.NumThreads = 4 + } return o, nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go index 5cc7f8873b6..16360c898b5 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/s3ng.go @@ -44,7 +44,16 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { return nil, fmt.Errorf("S3 configuration incomplete") } - bs, err := blobstore.New(o.S3Endpoint, o.S3Region, o.S3Bucket, o.S3AccessKey, o.S3SecretKey) + defaultPutOptions := blobstore.Options{ + DisableContentSha256: o.DisableContentSha256, + DisableMultipart: o.DisableMultipart, + SendContentMd5: o.SendContentMd5, + ConcurrentStreamParts: o.ConcurrentStreamParts, + NumThreads: o.NumThreads, + PartSize: o.PartSize, + } + + bs, err := blobstore.New(o.S3Endpoint, o.S3Region, o.S3Bucket, o.S3AccessKey, o.S3SecretKey, defaultPutOptions) if err != nil { return nil, err } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go index c39cae9e0e4..c8ee7b9fb65 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/metadata/cs3.go @@ -165,7 +165,9 @@ func (cs3 *CS3) Upload(ctx context.Context, req UploadRequest) (*UploadResponse, } if len(req.IfNoneMatch) > 0 { if req.IfNoneMatch[0] == "*" { - ifuReq.Options = &provider.InitiateFileUploadRequest_IfNotExist{} + ifuReq.Options = &provider.InitiateFileUploadRequest_IfNotExist{ + IfNotExist: true, + } } // else { // the http upload will carry all if-not-match etags diff --git a/vendor/modules.txt b/vendor/modules.txt index 51af73c31d3..e9086c7098d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -359,7 +359,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.18.1-0.20240208163049-039a779e377b +# github.com/cs3org/reva/v2 v2.18.1-0.20240220100621-283b39bc20e6 ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime