From 0e6737ddda78ed7f1c352b1da5621e000479b15a Mon Sep 17 00:00:00 2001 From: Amal Thundiyil Date: Mon, 2 May 2022 15:40:55 +0530 Subject: [PATCH] ci: fix build --- internal/grpc/services/gateway/gateway.go | 30 ++--- .../http/services/owncloud/ocdav/ocdav.go | 6 +- pkg/rgrpc/todo/pool/pool.go | 14 ++- pkg/storage/fs/owncloud/owncloud.go | 4 +- pkg/storage/fs/owncloudsql/owncloudsql.go | 4 +- .../utils/decomposedfs/options/options.go | 23 ++-- pkg/storage/utils/eosfs/config.go | 110 +++++++++--------- 7 files changed, 99 insertions(+), 92 deletions(-) diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 81d8a298890..b159cb70153 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -58,24 +58,24 @@ type config struct { DataTxEndpoint string `mapstructure:"datatx"` DataGatewayEndpoint string `mapstructure:"datagateway"` PermissionsEndpoint string `mapstructure:"permissionssvc"` - CommitShareToStorageGrant bool `mapstructure:"commit_share_to_storage_grant"` - CommitShareToStorageRef bool `mapstructure:"commit_share_to_storage_ref"` - DisableHomeCreationOnLogin bool `mapstructure:"disable_home_creation_on_login"` TransferSharedSecret string `mapstructure:"transfer_shared_secret"` - TransferExpires int64 `mapstructure:"transfer_expires"` TokenManager string `mapstructure:"token_manager"` // ShareFolder is the location where to create shares in the recipient's storage provider. - ShareFolder string `mapstructure:"share_folder"` - DataTransfersFolder string `mapstructure:"data_transfers_folder"` - HomeMapping string `mapstructure:"home_mapping"` - TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` - EtagCacheTTL int `mapstructure:"etag_cache_ttl"` - AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent - CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"` - CACertFile string `mapstructure:"ca_certfile"` - MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` - Insecure bool `mapstructure:"insecure"` - SkipVerify bool `mapstructure:"skip_verify"` + ShareFolder string `mapstructure:"share_folder"` + DataTransfersFolder string `mapstructure:"data_transfers_folder"` + HomeMapping string `mapstructure:"home_mapping"` + CACertFile string `mapstructure:"ca_certfile"` + TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` + AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent + TransferExpires int64 `mapstructure:"transfer_expires"` + EtagCacheTTL int `mapstructure:"etag_cache_ttl"` + CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"` + MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` + CommitShareToStorageGrant bool `mapstructure:"commit_share_to_storage_grant"` + CommitShareToStorageRef bool `mapstructure:"commit_share_to_storage_ref"` + DisableHomeCreationOnLogin bool `mapstructure:"disable_home_creation_on_login"` + Insecure bool `mapstructure:"insecure"` + SkipVerify bool `mapstructure:"skip_verify"` } // sets defaults diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index f0720773b50..79c74494477 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -98,14 +98,14 @@ type Config struct { // /users///docs WebdavNamespace string `mapstructure:"webdav_namespace"` GatewaySvc string `mapstructure:"gatewaysvc"` - Timeout int64 `mapstructure:"timeout"` // If true, HTTP COPY will expect the HTTP-TPC (third-party copy) headers - EnableHTTPTpc bool `mapstructure:"enable_http_tpc"` PublicURL string `mapstructure:"public_url"` FavoriteStorageDriver string `mapstructure:"favorite_storage_driver"` - FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"` CACertFile string `mapstructure:"ca_certfile"` + FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"` + Timeout int64 `mapstructure:"timeout"` MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` + EnableHTTPTpc bool `mapstructure:"enable_http_tpc"` Insecure bool `mapstructure:"insecure"` SkipVerify bool `mapstructure:"skip_verify"` } diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 38a9119349b..2673383aa04 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -106,7 +106,8 @@ func getConnectionOptions(options Options) ([]grpc.DialOption, error) { opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions( - grpc.MaxCallRecvMsgSize(options.MaxCallRecvMsgSize), + // TODO @amal-thundiyil: change this to options.MaxCallRecvMsgSize + grpc.MaxCallRecvMsgSize(10240000), ), grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor( otelgrpc.WithTracerProvider( @@ -137,11 +138,13 @@ func getConnectionOptions(options Options) ([]grpc.DialOption, error) { func getCredentials(options Options) (credentials.TransportCredentials, error) { var creds credentials.TransportCredentials - if options.Insecure && options.CACertFile != "" { + switch { + case options.Insecure && options.CACertFile != "": return nil, errors.New("can't set insecure and ca_certfile at the same time") - } else if true { + // TODO @amal-thundiyil: change this to options.Insecure + case true: creds = insecure.NewCredentials() - } else { + case options.CACertFile != "": b, err := ioutil.ReadFile(options.CACertFile) if err != nil { return nil, err @@ -155,7 +158,8 @@ func getCredentials(options Options) (credentials.TransportCredentials, error) { RootCAs: cp, } creds = credentials.NewTLS(tlsconf) - + default: + return nil, errors.New("invalid grpc security configuration") } return creds, nil } diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index bb5bc2d205a..f8e22f3d18c 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -116,11 +116,11 @@ type config struct { ShareFolder string `mapstructure:"share_folder"` UserLayout string `mapstructure:"user_layout"` Redis string `mapstructure:"redis"` - EnableHome bool `mapstructure:"enable_home"` - Scan bool `mapstructure:"scan"` UserProviderEndpoint string `mapstructure:"userprovidersvc"` CACertFile string `mapstructure:"ca_certfile"` MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` + EnableHome bool `mapstructure:"enable_home"` + Scan bool `mapstructure:"scan"` Insecure bool `mapstructure:"insecure"` SkipVerify bool `mapstructure:"skip_verify"` } diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 908d75aa827..cd400695259 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -112,15 +112,15 @@ type config struct { DeprecatedShareDirectory string `mapstructure:"sharedirectory"` ShareFolder string `mapstructure:"share_folder"` UserLayout string `mapstructure:"user_layout"` - EnableHome bool `mapstructure:"enable_home"` UserProviderEndpoint string `mapstructure:"userprovidersvc"` DbUsername string `mapstructure:"dbusername"` DbPassword string `mapstructure:"dbpassword"` DbHost string `mapstructure:"dbhost"` - DbPort int `mapstructure:"dbport"` DbName string `mapstructure:"dbname"` CACertFile string `mapstructure:"ca_certfile"` + DbPort int `mapstructure:"dbport"` MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` + EnableHome bool `mapstructure:"enable_home"` Insecure bool `mapstructure:"insecure"` SkipVerify bool `mapstructure:"skip_verify"` } diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index 12c43d867d4..e4623fdd863 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -40,15 +40,6 @@ type Options struct { // TODO NodeLayout option to save nodes as eg. nodes/1d/d8/1dd84abf-9466-4e14-bb86-02fc4ea3abcf ShareFolder string `mapstructure:"share_folder"` - // EnableHome enables the creation of home directories. - EnableHome bool `mapstructure:"enable_home"` - - // propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node - TreeTimeAccounting bool `mapstructure:"treetime_accounting"` - - // propagate size changes as treesize - TreeSizeAccounting bool `mapstructure:"treesize_accounting"` - // set an owner for the root node Owner string `mapstructure:"owner"` OwnerIDP string `mapstructure:"owner_idp"` @@ -58,8 +49,18 @@ type Options struct { CACertFile string `mapstructure:"ca_certfile"` MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` - Insecure bool `mapstructure:"insecure"` - SkipVerify bool `mapstructure:"skip_verify"` + // EnableHome enables the creation of home directories. + + EnableHome bool `mapstructure:"enable_home"` + + // propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node + TreeTimeAccounting bool `mapstructure:"treetime_accounting"` + + // propagate size changes as treesize + TreeSizeAccounting bool `mapstructure:"treesize_accounting"` + + Insecure bool `mapstructure:"insecure"` + SkipVerify bool `mapstructure:"skip_verify"` } // New returns a new Options instance for the given configuration diff --git a/pkg/storage/utils/eosfs/config.go b/pkg/storage/utils/eosfs/config.go index 36b38f8b528..827e53f6322 100644 --- a/pkg/storage/utils/eosfs/config.go +++ b/pkg/storage/utils/eosfs/config.go @@ -26,12 +26,6 @@ type Config struct { // QuotaNode for storing quota information QuotaNode string `mapstructure:"quota_node"` - // DefaultQuotaBytes sets the default maximum bytes available for a user - DefaultQuotaBytes uint64 `mapstructure:"default_quota_bytes"` - - // DefaultQuotaFiles sets the default maximum files available for a user - DefaultQuotaFiles uint64 `mapstructure:"default_quota_files"` - // ShadowNamespace for storing shadow data ShadowNamespace string `mapstructure:"shadow_namespace"` @@ -77,6 +71,60 @@ type Config struct { // /eos/user//docs UserLayout string `mapstructure:"user_layout"` + // GatewaySvc stores the endpoint at which the GRPC gateway is exposed. + GatewaySvc string `mapstructure:"gatewaysvc"` + + // HTTP connections to EOS: client certificate (usually a X509 host certificate) + ClientCertFile string `mapstructure:"http_client_certfile"` + // HTTP connections to EOS: client certificate key (usually a X509 host certificate) + ClientKeyFile string `mapstructure:"http_client_keyfile"` + // HTTP connections to EOS: CA directories + ClientCADirs string `mapstructure:"http_client_cadirs"` + // HTTP connections to EOS: CA files + ClientCAFiles string `mapstructure:"http_client_cafiles"` + + // GRPCAuthkey is the key that authorizes this client to connect to the GRPC service + // It's unclear whether this will be the final solution + GRPCAuthkey string `mapstructure:"grpc_auth_key"` + + // URI of the EOS MGM grpc server + // Default is empty + GrpcURI string `mapstructure:"master_grpc_uri"` + + CACertFile string `mapstructure:"ca_certfile"` + + // DefaultQuotaBytes sets the default maximum bytes available for a user + DefaultQuotaBytes uint64 `mapstructure:"default_quota_bytes"` + + // DefaultQuotaFiles sets the default maximum files available for a user + DefaultQuotaFiles uint64 `mapstructure:"default_quota_files"` + + // Size of the cache used to store user ID and UID resolution. + // Default value is 1000000. + UserIDCacheSize int `mapstructure:"user_id_cache_size"` + + // The depth, starting from root, that we'll parse directories to lookup the + // owner and warm up the cache. For example, for a layout of {{substr 0 1 .Username}}/{{.Username}} + // and a depth of 2, we'll lookup each user's home directory. + // Default value is 2. + UserIDCacheWarmupDepth int `mapstructure:"user_id_cache_warmup_depth"` + + // HTTP connections to EOS: max number of idle conns + MaxIdleConns int `mapstructure:"max_idle_conns"` + + // HTTP connections to EOS: max number of conns per host + MaxConnsPerHost int `mapstructure:"max_conns_per_host"` + + // HTTP connections to EOS: max number of idle conns per host + MaxIdleConnsPerHost int `mapstructure:"max_idle_conns_per_host"` + + // HTTP connections to EOS: idle conections TTL + IdleConnTimeout int `mapstructure:"idle_conn_timeout"` + + // TokenExpiry stores in seconds the time after which generated tokens will expire + // Default is 3600 + TokenExpiry int + MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` // Enables logging of the commands executed // Defaults to false EnableLogging bool `mapstructure:"enable_logging"` @@ -101,27 +149,6 @@ type Config struct { // UseGRPC controls whether we spawn eosclient processes or use GRPC to connect to EOS. UseGRPC bool `mapstructure:"use_grpc"` - // GatewaySvc stores the endpoint at which the GRPC gateway is exposed. - GatewaySvc string `mapstructure:"gatewaysvc"` - - // GRPCAuthkey is the key that authorizes this client to connect to the GRPC service - // It's unclear whether this will be the final solution - GRPCAuthkey string `mapstructure:"grpc_auth_key"` - - // URI of the EOS MGM grpc server - // Default is empty - GrpcURI string `mapstructure:"master_grpc_uri"` - - // Size of the cache used to store user ID and UID resolution. - // Default value is 1000000. - UserIDCacheSize int `mapstructure:"user_id_cache_size"` - - // The depth, starting from root, that we'll parse directories to lookup the - // owner and warm up the cache. For example, for a layout of {{substr 0 1 .Username}}/{{.Username}} - // and a depth of 2, we'll lookup each user's home directory. - // Default value is 2. - UserIDCacheWarmupDepth int `mapstructure:"user_id_cache_warmup_depth"` - // Normally the eosgrpc plugin streams data on the fly. // Setting this to true will make reva use the temp cachedirectory // as intermediate step for read operations @@ -144,32 +171,7 @@ type Config struct { // revisions-related operations. ImpersonateOwnerforRevisions bool `mapstructure:"impersonate_owner_for_revisions"` - // HTTP connections to EOS: max number of idle conns - MaxIdleConns int `mapstructure:"max_idle_conns"` - - // HTTP connections to EOS: max number of conns per host - MaxConnsPerHost int `mapstructure:"max_conns_per_host"` - - // HTTP connections to EOS: max number of idle conns per host - MaxIdleConnsPerHost int `mapstructure:"max_idle_conns_per_host"` - - // HTTP connections to EOS: idle conections TTL - IdleConnTimeout int `mapstructure:"idle_conn_timeout"` + Insecure bool `mapstructure:"insecure"` - // HTTP connections to EOS: client certificate (usually a X509 host certificate) - ClientCertFile string `mapstructure:"http_client_certfile"` - // HTTP connections to EOS: client certificate key (usually a X509 host certificate) - ClientKeyFile string `mapstructure:"http_client_keyfile"` - // HTTP connections to EOS: CA directories - ClientCADirs string `mapstructure:"http_client_cadirs"` - // HTTP connections to EOS: CA files - ClientCAFiles string `mapstructure:"http_client_cafiles"` - - // TokenExpiry stores in seconds the time after which generated tokens will expire - // Default is 3600 - TokenExpiry int - CACertFile string `mapstructure:"ca_certfile"` - MaxCallRecvMsgSize int `mapstructure:"client_recv_msg_size"` - Insecure bool `mapstructure:"insecure"` - SkipVerify bool `mapstructure:"skip_verify"` + SkipVerify bool `mapstructure:"skip_verify"` }