From 99db7c174bb809917d01edbdbb131c5eea56f8b7 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 28 Jul 2020 17:10:41 +0200 Subject: [PATCH] Add UID and GID to the user object from EOS fs (#995) --- changelog/unreleased/uid-gid-user-object.md | 7 + .../en/docs/config/packages/auth/_index.md | 7 + .../config/packages/auth/manager/_index.md | 7 + .../packages/auth/manager/oidc/_index.md | 50 +++ .../config/packages/storage/fs/eos/_index.md | 8 + .../packages/storage/fs/eoshome/_index.md | 8 + .../packages/user/manager/rest/_index.md | 20 +- go.mod | 4 +- go.sum | 8 +- internal/grpc/services/gateway/appprovider.go | 2 - .../grpc/services/gateway/userprovider.go | 16 + .../services/gateway/usershareprovider.go | 2 - .../storageprovider/storageprovider.go | 20 +- .../services/userprovider/userprovider.go | 20 +- pkg/auth/manager/oidc/oidc.go | 32 +- pkg/eosclient/eosclient.go | 339 ++++++--------- pkg/storage/fs/eos/eos.go | 3 + pkg/storage/fs/eoshome/eoshome.go | 3 + pkg/storage/utils/acl/acl.go | 6 + pkg/storage/utils/eosfs/eosfs.go | 404 ++++++++++++++---- pkg/storage/utils/eosfs/upload.go | 15 +- pkg/user/manager/demo/demo.go | 53 +++ pkg/user/manager/demo/demo_test.go | 36 +- pkg/user/manager/json/json.go | 27 ++ pkg/user/manager/json/json_test.go | 28 +- pkg/user/manager/ldap/ldap.go | 4 + pkg/user/manager/rest/cache.go | 149 +++++-- pkg/user/manager/rest/rest.go | 174 ++++++-- pkg/user/user.go | 1 + 29 files changed, 1051 insertions(+), 402 deletions(-) create mode 100644 changelog/unreleased/uid-gid-user-object.md create mode 100644 docs/content/en/docs/config/packages/auth/_index.md create mode 100644 docs/content/en/docs/config/packages/auth/manager/_index.md create mode 100644 docs/content/en/docs/config/packages/auth/manager/oidc/_index.md diff --git a/changelog/unreleased/uid-gid-user-object.md b/changelog/unreleased/uid-gid-user-object.md new file mode 100644 index 0000000000..ae147afe0e --- /dev/null +++ b/changelog/unreleased/uid-gid-user-object.md @@ -0,0 +1,7 @@ +Enhancement: Add UID and GID to the user object from user package + +Currently, the UID and GID for users need to be read from the local system which +requires local users to be present. This change retrieves that information from +the user and auth packages and adds methods to retrieve it. + +https://github.com/cs3org/reva/pull/995 diff --git a/docs/content/en/docs/config/packages/auth/_index.md b/docs/content/en/docs/config/packages/auth/_index.md new file mode 100644 index 0000000000..2211edf59e --- /dev/null +++ b/docs/content/en/docs/config/packages/auth/_index.md @@ -0,0 +1,7 @@ +--- +title: "auth" +linkTitle: "auth" +weight: 10 +description: > + Configuration for the auth service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/auth/manager/_index.md b/docs/content/en/docs/config/packages/auth/manager/_index.md new file mode 100644 index 0000000000..ddc3cb8170 --- /dev/null +++ b/docs/content/en/docs/config/packages/auth/manager/_index.md @@ -0,0 +1,7 @@ +--- +title: "manager" +linkTitle: "manager" +weight: 10 +description: > + Configuration for the manager service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md new file mode 100644 index 0000000000..0df3479857 --- /dev/null +++ b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md @@ -0,0 +1,50 @@ +--- +title: "oidc" +linkTitle: "oidc" +weight: 10 +description: > + Configuration for the oidc service +--- + +# _struct: config_ + +{{% dir name="insecure" type="bool" default=false %}} +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L50) +{{< highlight toml >}} +[auth.manager.oidc] +insecure = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="issuer" type="string" default="" %}} +The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L51) +{{< highlight toml >}} +[auth.manager.oidc] +issuer = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="id_claim" type="string" default="sub" %}} +The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L52) +{{< highlight toml >}} +[auth.manager.oidc] +id_claim = "sub" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="uid_claim" type="string" default="" %}} +The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L53) +{{< highlight toml >}} +[auth.manager.oidc] +uid_claim = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="gid_claim" type="string" default="" %}} +The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L54) +{{< highlight toml >}} +[auth.manager.oidc] +gid_claim = "" +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/storage/fs/eos/_index.md b/docs/content/en/docs/config/packages/storage/fs/eos/_index.md index 8e5f1c83a9..2f29a70cab 100644 --- a/docs/content/en/docs/config/packages/storage/fs/eos/_index.md +++ b/docs/content/en/docs/config/packages/storage/fs/eos/_index.md @@ -136,3 +136,11 @@ use_keytab = false {{< /highlight >}} {{% /dir %}} +{{% dir name="gatewaysvc" type="string" default="0.0.0.0:19000" %}} +GatewaySvc stores the endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L94) +{{< highlight toml >}} +[storage.fs.eos] +gatewaysvc = "0.0.0.0:19000" +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md b/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md index 033b477861..8f29afcf8f 100644 --- a/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md +++ b/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md @@ -144,3 +144,11 @@ use_keytab = false {{< /highlight >}} {{% /dir %}} +{{% dir name="gatewaysvc" type="string" default="0.0.0.0:19000" %}} +GatewaySvc stores the endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L100) +{{< highlight toml >}} +[storage.fs.eoshome] +gatewaysvc = "0.0.0.0:19000" +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/user/manager/rest/_index.md b/docs/content/en/docs/config/packages/user/manager/rest/_index.md index 6a5e781717..6999e77085 100644 --- a/docs/content/en/docs/config/packages/user/manager/rest/_index.md +++ b/docs/content/en/docs/config/packages/user/manager/rest/_index.md @@ -8,16 +8,16 @@ description: > # _struct: config_ -{{% dir name="redis" type="string" default=":6379" %}} -The port on which the redis server is running [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L68) +{{% dir name="redis_address" type="string" default="localhost:6379" %}} +The address at which the redis server is running [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L68) {{< highlight toml >}} [user.manager.rest] -redis = ":6379" +redis_address = "localhost:6379" {{< /highlight >}} {{% /dir %}} {{% dir name="user_groups_cache_expiration" type="int" default=5 %}} -The time in minutes for which the groups to which a user belongs would be cached [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L70) +The time in minutes for which the groups to which a user belongs would be cached [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L74) {{< highlight toml >}} [user.manager.rest] user_groups_cache_expiration = 5 @@ -25,7 +25,7 @@ user_groups_cache_expiration = 5 {{% /dir %}} {{% dir name="id_provider" type="string" default="http://cernbox.cern.ch" %}} -The OIDC Provider [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L72) +The OIDC Provider [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L76) {{< highlight toml >}} [user.manager.rest] id_provider = "http://cernbox.cern.ch" @@ -33,7 +33,7 @@ id_provider = "http://cernbox.cern.ch" {{% /dir %}} {{% dir name="api_base_url" type="string" default="https://authorization-service-api-dev.web.cern.ch/api/v1.0" %}} -Base API Endpoint [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L74) +Base API Endpoint [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L78) {{< highlight toml >}} [user.manager.rest] api_base_url = "https://authorization-service-api-dev.web.cern.ch/api/v1.0" @@ -41,7 +41,7 @@ api_base_url = "https://authorization-service-api-dev.web.cern.ch/api/v1.0" {{% /dir %}} {{% dir name="client_id" type="string" default="-" %}} -Client ID needed to authenticate [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L76) +Client ID needed to authenticate [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L80) {{< highlight toml >}} [user.manager.rest] client_id = "-" @@ -49,7 +49,7 @@ client_id = "-" {{% /dir %}} {{% dir name="client_secret" type="string" default="-" %}} -Client Secret [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L78) +Client Secret [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L82) {{< highlight toml >}} [user.manager.rest] client_secret = "-" @@ -57,7 +57,7 @@ client_secret = "-" {{% /dir %}} {{% dir name="oidc_token_endpoint" type="string" default="https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" %}} -Endpoint to generate token to access the API [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L81) +Endpoint to generate token to access the API [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L85) {{< highlight toml >}} [user.manager.rest] oidc_token_endpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" @@ -65,7 +65,7 @@ oidc_token_endpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/ {{% /dir %}} {{% dir name="target_api" type="string" default="authorization-service-api" %}} -The target application for which token needs to be generated [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L83) +The target application for which token needs to be generated [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/rest/rest.go#L87) {{< highlight toml >}} [user.manager.rest] target_api = "authorization-service-api" diff --git a/go.mod b/go.mod index e6dbf9491b..ac1c8093f6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/cheggaaa/pb v1.0.28 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200626150132-28a40e643719 - github.com/cs3org/go-cs3apis v0.0.0-20200709064917-d96c5f2a42ad + github.com/cs3org/go-cs3apis v0.0.0-20200728114537-4efa23660dbe github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/go-ldap/ldap/v3 v3.2.3 @@ -19,7 +19,7 @@ require ( github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.3.0+incompatible github.com/golang/protobuf v1.4.2 - github.com/gomodule/redigo v2.0.0+incompatible + github.com/gomodule/redigo v1.8.2 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.1.1 diff --git a/go.sum b/go.sum index 5b648a56a0..6ade0ce0e5 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,10 @@ github.com/cs3org/cato v0.0.0-20200626150132-28a40e643719 h1:3vDKYhsyWSbrtX67i66 github.com/cs3org/cato v0.0.0-20200626150132-28a40e643719/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20200709064917-d96c5f2a42ad h1:XxB0h+UKILRKdr+WgPJaOfW8duVPeVKq/18aip5D/Ws= github.com/cs3org/go-cs3apis v0.0.0-20200709064917-d96c5f2a42ad/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20200720081540-0d96aec81a2e h1:Q1GsuqKBo74Z6WNkUTVmyCATf7WwaTk8Fyx3Xw4CrU4= +github.com/cs3org/go-cs3apis v0.0.0-20200720081540-0d96aec81a2e/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20200728114537-4efa23660dbe h1:CQ/Grq7oVFqwiUg4VA/T+fl3JHZKEyo/RcTE7C23rW4= +github.com/cs3org/go-cs3apis v0.0.0-20200728114537-4efa23660dbe/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -362,8 +366,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= diff --git a/internal/grpc/services/gateway/appprovider.go b/internal/grpc/services/gateway/appprovider.go index b7bee4f269..003dbabc93 100644 --- a/internal/grpc/services/gateway/appprovider.go +++ b/internal/grpc/services/gateway/appprovider.go @@ -54,14 +54,12 @@ func (s *svc) OpenFileInAppProvider(ctx context.Context, req *providerpb.OpenFil } statRes, err := c.Stat(ctx, statReq) - if err != nil { log.Err(err).Msg("gateway: error calling Stat for the share resource path:" + req.Ref.GetPath()) return &providerpb.OpenFileInAppProviderResponse{ Status: status.NewInternal(ctx, err, "gateway: error calling Stat for the share resource id"), }, nil } - if statRes.Status.Code != rpc.Code_CODE_OK { err := status.NewErrorFromCode(statRes.Status.GetCode(), "gateway") log.Err(err).Msg("gateway: error calling Stat for the share resource id:" + req.Ref.GetPath()) diff --git a/internal/grpc/services/gateway/userprovider.go b/internal/grpc/services/gateway/userprovider.go index 219facabf8..87c9e354c1 100644 --- a/internal/grpc/services/gateway/userprovider.go +++ b/internal/grpc/services/gateway/userprovider.go @@ -43,6 +43,22 @@ func (s *svc) GetUser(ctx context.Context, req *user.GetUserRequest) (*user.GetU return res, nil } +func (s *svc) GetUserByClaim(ctx context.Context, req *user.GetUserByClaimRequest) (*user.GetUserByClaimResponse, error) { + c, err := pool.GetUserProviderServiceClient(s.c.UserProviderEndpoint) + if err != nil { + return &user.GetUserByClaimResponse{ + Status: status.NewInternal(ctx, err, "error getting auth client"), + }, nil + } + + res, err := c.GetUserByClaim(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling GetUserByClaim") + } + + return res, nil +} + func (s *svc) FindUsers(ctx context.Context, req *user.FindUsersRequest) (*user.FindUsersResponse, error) { c, err := pool.GetUserProviderServiceClient(s.c.UserProviderEndpoint) if err != nil { diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index 4d430f0521..4bb6aff17c 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -41,12 +41,10 @@ func (s *svc) CreateShare(ctx context.Context, req *collaboration.CreateShareReq Status: status.NewInternal(ctx, err, "error getting user share provider client"), }, nil } - res, err := c.CreateShare(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling CreateShare") } - if res.Status.Code != rpc.Code_CODE_OK { return res, nil } diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 764acc7365..04b79ecb17 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -688,7 +688,25 @@ func (s *service) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleRe } func (s *service) ListGrants(ctx context.Context, req *provider.ListGrantsRequest) (*provider.ListGrantsResponse, error) { - return nil, nil + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + return &provider.ListGrantsResponse{ + Status: status.NewInternal(ctx, err, "error unwrapping path"), + }, nil + } + + grants, err := s.storage.ListGrants(ctx, newRef) + if err != nil { + return &provider.ListGrantsResponse{ + Status: status.NewInternal(ctx, err, "error listing ACLs"), + }, nil + } + + res := &provider.ListGrantsResponse{ + Status: status.NewOK(ctx), + Grants: grants, + } + return res, nil } func (s *service) AddGrant(ctx context.Context, req *provider.AddGrantRequest) (*provider.AddGrantResponse, error) { diff --git a/internal/grpc/services/userprovider/userprovider.go b/internal/grpc/services/userprovider/userprovider.go index 24fda91a89..46f31eb9da 100644 --- a/internal/grpc/services/userprovider/userprovider.go +++ b/internal/grpc/services/userprovider/userprovider.go @@ -104,7 +104,7 @@ func (s *service) GetUser(ctx context.Context, req *userpb.GetUserRequest) (*use // TODO(labkode): check for not found. err = errors.Wrap(err, "userprovidersvc: error getting user") res := &userpb.GetUserResponse{ - Status: status.NewInternal(ctx, err, "error authenticating user"), + Status: status.NewInternal(ctx, err, "error getting user"), } return res, nil } @@ -116,6 +116,24 @@ func (s *service) GetUser(ctx context.Context, req *userpb.GetUserRequest) (*use return res, nil } +func (s *service) GetUserByClaim(ctx context.Context, req *userpb.GetUserByClaimRequest) (*userpb.GetUserByClaimResponse, error) { + user, err := s.usermgr.GetUserByClaim(ctx, req.Claim, req.Value) + if err != nil { + // TODO(labkode): check for not found. + err = errors.Wrap(err, "userprovidersvc: error getting user by claim") + res := &userpb.GetUserByClaimResponse{ + Status: status.NewInternal(ctx, err, "error getting user by claim"), + } + return res, nil + } + + res := &userpb.GetUserByClaimResponse{ + Status: status.NewOK(ctx), + User: user, + } + return res, nil +} + func (s *service) FindUsers(ctx context.Context, req *userpb.FindUsersRequest) (*userpb.FindUsersResponse, error) { users, err := s.usermgr.FindUsers(ctx, req.Filter) if err != nil { diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index ab6941cc18..8e345478b9 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -27,6 +27,7 @@ import ( oidc "github.com/coreos/go-oidc" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/rhttp" @@ -46,9 +47,11 @@ type mgr struct { } type config struct { - Insecure bool `mapstructure:"insecure"` - Issuer string `mapstructure:"issuer"` - IDClaim string `mapstructure:"id_claim"` + Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` + Issuer string `mapstructure:"issuer" docs:";The issuer of the OIDC token."` + IDClaim string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."` + UIDClaim string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."` + GIDClaim string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."` } func (c *config) init() { @@ -120,6 +123,28 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) return nil, fmt.Errorf("no \"preferred_username\" or \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope") } + opaqueObj := &types.Opaque{ + Map: map[string]*types.OpaqueEntry{}, + } + if am.c.UIDClaim != "" { + uid, ok := claims[am.c.UIDClaim] + if ok { + opaqueObj.Map["uid"] = &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", uid)), + } + } + } + if am.c.GIDClaim != "" { + gid, ok := claims[am.c.GIDClaim] + if ok { + opaqueObj.Map["gid"] = &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", gid)), + } + } + } + u := &user.User{ Id: &user.UserId{ OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id @@ -134,6 +159,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) Mail: claims["email"].(string), MailVerified: claims["email_verified"].(bool), DisplayName: claims["name"].(string), + Opaque: opaqueObj, } return u, nil diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 21e5c44388..43cbf6c04a 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -26,7 +26,6 @@ import ( "io/ioutil" "os" "os/exec" - gouser "os/user" "path" "strconv" "strings" @@ -41,14 +40,18 @@ import ( ) const ( - rootUser = "root" versionPrefix = ".sys.v#." + + versionAquamarine = eosVersion("aquamarine") + versionCitrine = eosVersion("citrine") ) // AttrType is the type of extended attribute, // either system (sys) or user (user). type AttrType uint32 +type eosVersion string + const ( // SystemAttr is the system extended attribute. SystemAttr AttrType = iota @@ -159,13 +162,6 @@ func New(opt *Options) *Client { return c } -func (c *Client) getUnixUser(username string) (*gouser.User, error) { - if c.opt.ForceSingleUserMode { - username = c.opt.SingleUsername - } - return gouser.Lookup(username) -} - // exec executes the command and returns the stdout, stderr and return code func (c *Client) execute(ctx context.Context, cmd *exec.Cmd) (string, string, error) { log := appctx.GetLogger(ctx) @@ -269,76 +265,99 @@ func (c *Client) executeEOS(ctx context.Context, cmd *exec.Cmd) (string, string, return outBuf.String(), errBuf.String(), err } -// AddACL adds an new acl to EOS with the given aclType. -func (c *Client) AddACL(ctx context.Context, username, path string, a *acl.Entry) error { - acls, err := c.getACLForPath(ctx, username, path) +func (c *Client) getVersion(ctx context.Context, rootUID, rootGID string) (eosVersion, error) { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "version") + stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { - return err + return "", err } + return c.parseVersion(ctx, stdout), nil +} - // since EOS Citrine ACLs are is stored with uid, we need to convert username to uid - // only for users. - if a.Type == acl.TypeUser { - a.Qualifier, err = getUID(a.Qualifier) - if err != nil { - return err +func (c *Client) parseVersion(ctx context.Context, raw string) eosVersion { + var serverVersion string + rawLines := strings.Split(raw, "\n") + for _, rl := range rawLines { + if rl == "" { + continue + } + if strings.HasPrefix(rl, "EOS_SERVER_VERSION") { + serverVersion = strings.Split(strings.Split(rl, " ")[0], "=")[1] + break } } - err = acls.SetEntry(a.Type, a.Qualifier, a.Permissions) - if err != nil { - return err + + if strings.HasPrefix(serverVersion, "4.") { + return versionCitrine } - sysACL := acls.Serialize() + return versionAquamarine +} - // setting of the sys.acl is only possible from root user - unixUser, err := c.getUnixUser(rootUser) +// AddACL adds an new acl to EOS with the given aclType. +func (c *Client) AddACL(ctx context.Context, uid, gid, rootUID, rootGID, path string, a *acl.Entry) error { + version, err := c.getVersion(ctx, rootUID, rootGID) if err != nil { return err } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "attr", "-r", "set", fmt.Sprintf("sys.acl=%s", sysACL), path) + var cmd *exec.Cmd + if version == versionCitrine { + sysACL := a.CitrineSerialize() + cmd = exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "acl", "--sys", "--recursive", sysACL, path) + } else { + acls, err := c.getACLForPath(ctx, uid, gid, path) + if err != nil { + return err + } + + err = acls.SetEntry(a.Type, a.Qualifier, a.Permissions) + if err != nil { + return err + } + sysACL := acls.Serialize() + cmd = exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "attr", "-r", "set", fmt.Sprintf("sys.acl=%s", sysACL), path) + } + _, _, err = c.executeEOS(ctx, cmd) return err } // RemoveACL removes the acl from EOS. -func (c *Client) RemoveACL(ctx context.Context, username, path string, aclType string, recipient string) error { - acls, err := c.getACLForPath(ctx, username, path) +func (c *Client) RemoveACL(ctx context.Context, uid, gid, rootUID, rootGID, path string, a *acl.Entry) error { + version, err := c.getVersion(ctx, rootUID, rootGID) if err != nil { return err } - // since EOS Citrine ACLs are stored with uid, we need to convert username to uid - if aclType == acl.TypeUser { - recipient, err = getUID(recipient) + var cmd *exec.Cmd + if version == versionCitrine { + sysACL := a.CitrineSerialize() + cmd = exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "acl", "--sys", "--recursive", sysACL, path) + } else { + acls, err := c.getACLForPath(ctx, uid, gid, path) if err != nil { return err } - } - acls.DeleteEntry(aclType, recipient) - sysACL := acls.Serialize() - // setting of the sys.acl is only possible from root user - unixUser, err := c.getUnixUser(rootUser) - if err != nil { - return err + acls.DeleteEntry(a.Type, a.Qualifier) + sysACL := acls.Serialize() + cmd = exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "attr", "-r", "set", fmt.Sprintf("sys.acl=%s", sysACL), path) } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "attr", "-r", "set", fmt.Sprintf("sys.acl=%s", sysACL), path) _, _, err = c.executeEOS(ctx, cmd) return err } // UpdateACL updates the EOS acl. -func (c *Client) UpdateACL(ctx context.Context, username, path string, a *acl.Entry) error { - return c.AddACL(ctx, username, path, a) +func (c *Client) UpdateACL(ctx context.Context, uid, gid, rootUID, rootGID, path string, a *acl.Entry) error { + return c.AddACL(ctx, uid, gid, rootUID, rootGID, path, a) } // GetACL for a file -func (c *Client) GetACL(ctx context.Context, username, path, aclType, target string) (*acl.Entry, error) { - acls, err := c.ListACLs(ctx, username, path) +func (c *Client) GetACL(ctx context.Context, uid, gid, path, aclType, target string) (*acl.Entry, error) { + acls, err := c.ListACLs(ctx, uid, gid, path) if err != nil { return nil, err } @@ -351,49 +370,23 @@ func (c *Client) GetACL(ctx context.Context, username, path, aclType, target str } -func getUsername(uid string) (string, error) { - user, err := gouser.LookupId(uid) - if err != nil { - return "", err - } - return user.Username, nil -} - -func getUID(username string) (string, error) { - user, err := gouser.Lookup(username) - if err != nil { - return "", err - } - return user.Uid, nil -} - // ListACLs returns the list of ACLs present under the given path. // EOS returns uids/gid for Citrine version and usernames for older versions. // For Citire we need to convert back the uid back to username. -func (c *Client) ListACLs(ctx context.Context, username, path string) ([]*acl.Entry, error) { - log := appctx.GetLogger(ctx) +func (c *Client) ListACLs(ctx context.Context, uid, gid, path string) ([]*acl.Entry, error) { - parsedACLs, err := c.getACLForPath(ctx, username, path) + parsedACLs, err := c.getACLForPath(ctx, uid, gid, path) if err != nil { return nil, err } - acls := []*acl.Entry{} - for _, acl := range parsedACLs.Entries { - // since EOS Citrine ACLs are is stored with uid, we need to convert uid to username - // TODO map group names as well if acl.Type == "g" ... - acl.Qualifier, err = getUsername(acl.Qualifier) - if err != nil { - log.Warn().Err(err).Str("path", path).Str("username", username).Str("qualifier", acl.Qualifier).Msg("cannot map qualifier to name") - continue - } - acls = append(acls, acl) - } - return acls, nil + // EOS Citrine ACLs are stored with uid. The UID will be resolved to the + // user opaque ID at the eosfs level. + return parsedACLs.Entries, nil } -func (c *Client) getACLForPath(ctx context.Context, username, path string) (*acl.ACLs, error) { - finfo, err := c.GetFileInfoByPath(ctx, username, path) +func (c *Client) getACLForPath(ctx context.Context, uid, gid, path string) (*acl.ACLs, error) { + finfo, err := c.GetFileInfoByPath(ctx, uid, gid, path) if err != nil { return nil, err } @@ -402,12 +395,8 @@ func (c *Client) getACLForPath(ctx context.Context, username, path string) (*acl } // GetFileInfoByInode returns the FileInfo by the given inode -func (c *Client) GetFileInfoByInode(ctx context.Context, username string, inode uint64) (*FileInfo, error) { - unixUser, err := c.getUnixUser(username) - if err != nil { - return nil, err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "file", "info", fmt.Sprintf("inode:%d", inode), "-m") +func (c *Client) GetFileInfoByInode(ctx context.Context, uid, gid string, inode uint64) (*FileInfo, error) { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "file", "info", fmt.Sprintf("inode:%d", inode), "-m") stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { return nil, err @@ -416,22 +405,18 @@ func (c *Client) GetFileInfoByInode(ctx context.Context, username string, inode } // SetAttr sets an extended attributes on a path. -func (c *Client) SetAttr(ctx context.Context, username string, attr *Attribute, recursive bool, path string) error { +func (c *Client) SetAttr(ctx context.Context, uid, gid string, attr *Attribute, recursive bool, path string) error { if !attr.isValid() { return errors.New("eos: attr is invalid: " + attr.serialize()) } - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } var cmd *exec.Cmd if recursive { - cmd = exec.CommandContext(ctx, "/usr/bin/eos", "-r", unixUser.Uid, unixUser.Gid, "attr", "-r", "set", attr.serialize(), path) + cmd = exec.CommandContext(ctx, "/usr/bin/eos", "-r", uid, gid, "attr", "-r", "set", attr.serialize(), path) } else { - cmd = exec.CommandContext(ctx, "/usr/bin/eos", "-r", unixUser.Uid, unixUser.Gid, "attr", "set", attr.serialize(), path) + cmd = exec.CommandContext(ctx, "/usr/bin/eos", "-r", uid, gid, "attr", "set", attr.serialize(), path) } - _, _, err = c.executeEOS(ctx, cmd) + _, _, err := c.executeEOS(ctx, cmd) if err != nil { return err } @@ -439,16 +424,12 @@ func (c *Client) SetAttr(ctx context.Context, username string, attr *Attribute, } // UnsetAttr unsets an extended attribute on a path. -func (c *Client) UnsetAttr(ctx context.Context, username string, attr *Attribute, path string) error { +func (c *Client) UnsetAttr(ctx context.Context, uid, gid string, attr *Attribute, path string) error { if !attr.isValid() { return errors.New("eos: attr is invalid: " + attr.serialize()) } - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, "/usr/bin/eos", "-r", unixUser.Uid, unixUser.Gid, "attr", "-r", "rm", fmt.Sprintf("%s.%s", attr.Type, attr.Key), path) - _, _, err = c.executeEOS(ctx, cmd) + cmd := exec.CommandContext(ctx, "/usr/bin/eos", "-r", uid, gid, "attr", "-r", "rm", fmt.Sprintf("%s.%s", attr.Type, attr.Key), path) + _, _, err := c.executeEOS(ctx, cmd) if err != nil { return err } @@ -456,12 +437,8 @@ func (c *Client) UnsetAttr(ctx context.Context, username string, attr *Attribute } // GetFileInfoByPath returns the FilInfo at the given path -func (c *Client) GetFileInfoByPath(ctx context.Context, username, path string) (*FileInfo, error) { - unixUser, err := c.getUnixUser(username) - if err != nil { - return nil, err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "file", "info", path, "-m") +func (c *Client) GetFileInfoByPath(ctx context.Context, uid, gid, path string) (*FileInfo, error) { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "file", "info", path, "-m") stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { return nil, err @@ -470,13 +447,8 @@ func (c *Client) GetFileInfoByPath(ctx context.Context, username, path string) ( } // GetQuota gets the quota of a user on the quota node defined by path -func (c *Client) GetQuota(ctx context.Context, username, path string) (*QuotaInfo, error) { - // setting of the sys.acl is only possible from root user - unixUser, err := c.getUnixUser(rootUser) - if err != nil { - return nil, err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "quota", "ls", "-u", username, "-m") +func (c *Client) GetQuota(ctx context.Context, username, rootUID, rootGID, path string) (*QuotaInfo, error) { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", rootUID, rootGID, "quota", "ls", "-u", username, "-m") stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { return nil, err @@ -485,87 +457,50 @@ func (c *Client) GetQuota(ctx context.Context, username, path string) (*QuotaInf } // Touch creates a 0-size,0-replica file in the EOS namespace. -func (c *Client) Touch(ctx context.Context, username, path string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - - cmd := exec.CommandContext(ctx, "/usr/bin/eos", "-r", unixUser.Uid, unixUser.Gid, "file", "touch", path) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) Touch(ctx context.Context, uid, gid, path string) error { + cmd := exec.CommandContext(ctx, "/usr/bin/eos", "-r", uid, gid, "file", "touch", path) + _, _, err := c.executeEOS(ctx, cmd) return err } // Chown given path -func (c *Client) Chown(ctx context.Context, username, chownUser, path string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - - unixChownUser, err := c.getUnixUser(chownUser) - if err != nil { - return err - } - - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "chown", unixChownUser.Uid+":"+unixChownUser.Gid, path) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) Chown(ctx context.Context, uid, gid, chownUID, chownGID, path string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "chown", chownUID+":"+chownGID, path) + _, _, err := c.executeEOS(ctx, cmd) return err } // Chmod given path -func (c *Client) Chmod(ctx context.Context, username, mode, path string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "chmod", mode, path) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) Chmod(ctx context.Context, uid, gid, mode, path string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "chmod", mode, path) + _, _, err := c.executeEOS(ctx, cmd) return err } // CreateDir creates a directory at the given path -func (c *Client) CreateDir(ctx context.Context, username, path string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "mkdir", "-p", path) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) CreateDir(ctx context.Context, uid, gid, path string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "mkdir", "-p", path) + _, _, err := c.executeEOS(ctx, cmd) return err } // Remove removes the resource at the given path -func (c *Client) Remove(ctx context.Context, username, path string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "rm", "-r", path) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) Remove(ctx context.Context, uid, gid, path string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "rm", "-r", path) + _, _, err := c.executeEOS(ctx, cmd) return err } // Rename renames the resource referenced by oldPath to newPath -func (c *Client) Rename(ctx context.Context, username, oldPath, newPath string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "file", "rename", oldPath, newPath) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) Rename(ctx context.Context, uid, gid, oldPath, newPath string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "file", "rename", oldPath, newPath) + _, _, err := c.executeEOS(ctx, cmd) return err } // List the contents of the directory given by path -func (c *Client) List(ctx context.Context, username, path string) ([]*FileInfo, error) { - unixUser, err := c.getUnixUser(username) - if err != nil { - return nil, err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "find", "--fileinfo", "--maxdepth", "1", path) +func (c *Client) List(ctx context.Context, uid, gid, path string) ([]*FileInfo, error) { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "find", "--fileinfo", "--maxdepth", "1", path) stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) @@ -574,17 +509,13 @@ func (c *Client) List(ctx context.Context, username, path string) ([]*FileInfo, } // Read reads a file from the mgm -func (c *Client) Read(ctx context.Context, username, path string) (io.ReadCloser, error) { - unixUser, err := c.getUnixUser(username) - if err != nil { - return nil, err - } +func (c *Client) Read(ctx context.Context, uid, gid, path string) (io.ReadCloser, error) { uuid := uuid.Must(uuid.NewV4()) rand := "eosread-" + uuid.String() localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", xrdPath, localTarget, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) - _, _, err = c.execute(ctx, cmd) + cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", xrdPath, localTarget, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s", uid, gid)) + _, _, err := c.execute(ctx, cmd) if err != nil { return nil, err } @@ -592,7 +523,7 @@ func (c *Client) Read(ctx context.Context, username, path string) (io.ReadCloser } // Write writes a stream to the mgm -func (c *Client) Write(ctx context.Context, username, path string, stream io.ReadCloser) error { +func (c *Client) Write(ctx context.Context, uid, gid, path string, stream io.ReadCloser) error { fd, err := ioutil.TempFile(c.opt.CacheDirectory, "eoswrite-") if err != nil { return err @@ -606,30 +537,22 @@ func (c *Client) Write(ctx context.Context, username, path string, stream io.Rea return err } - return c.WriteFile(ctx, username, path, fd.Name()) + return c.WriteFile(ctx, uid, gid, path, fd.Name()) } // WriteFile writes an existing file to the mgm -func (c *Client) WriteFile(ctx context.Context, username, path, source string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } +func (c *Client) WriteFile(ctx context.Context, uid, gid, path, source string) error { xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", source, xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) - _, _, err = c.execute(ctx, cmd) + cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", source, xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", uid, gid)) + _, _, err := c.execute(ctx, cmd) return err } // ListDeletedEntries returns a list of the deleted entries. -func (c *Client) ListDeletedEntries(ctx context.Context, username string) ([]*DeletedEntry, error) { - unixUser, err := c.getUnixUser(username) - if err != nil { - return nil, err - } +func (c *Client) ListDeletedEntries(ctx context.Context, uid, gid string) ([]*DeletedEntry, error) { // TODO(labkode): add protection if slave is configured and alive to count how many files are in the trashbin before // triggering the recycle ls call that could break the instance because of unavailable memory. - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "recycle", "ls", "-m") + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "recycle", "ls", "-m") stdout, _, err := c.executeEOS(ctx, cmd) if err != nil { return nil, err @@ -638,32 +561,24 @@ func (c *Client) ListDeletedEntries(ctx context.Context, username string) ([]*De } // RestoreDeletedEntry restores a deleted entry. -func (c *Client) RestoreDeletedEntry(ctx context.Context, username, key string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "recycle", "restore", key) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) RestoreDeletedEntry(ctx context.Context, uid, gid, key string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "recycle", "restore", key) + _, _, err := c.executeEOS(ctx, cmd) return err } // PurgeDeletedEntries purges all entries from the recycle bin. -func (c *Client) PurgeDeletedEntries(ctx context.Context, username string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "recycle", "purge") - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) PurgeDeletedEntries(ctx context.Context, uid, gid string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "recycle", "purge") + _, _, err := c.executeEOS(ctx, cmd) return err } // ListVersions list all the versions for a given file. -func (c *Client) ListVersions(ctx context.Context, username, p string) ([]*FileInfo, error) { +func (c *Client) ListVersions(ctx context.Context, uid, gid, p string) ([]*FileInfo, error) { basename := path.Base(p) versionFolder := path.Join(path.Dir(p), versionPrefix+basename) - finfos, err := c.List(ctx, username, versionFolder) + finfos, err := c.List(ctx, uid, gid, versionFolder) if err != nil { // we send back an empty list return []*FileInfo{}, nil @@ -672,21 +587,17 @@ func (c *Client) ListVersions(ctx context.Context, username, p string) ([]*FileI } // RollbackToVersion rollbacks a file to a previous version. -func (c *Client) RollbackToVersion(ctx context.Context, username, path, version string) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", unixUser.Uid, unixUser.Gid, "file", "versions", path, version) - _, _, err = c.executeEOS(ctx, cmd) +func (c *Client) RollbackToVersion(ctx context.Context, uid, gid, path, version string) error { + cmd := exec.CommandContext(ctx, c.opt.EosBinary, "-r", uid, gid, "file", "versions", path, version) + _, _, err := c.executeEOS(ctx, cmd) return err } // ReadVersion reads the version for the given file. -func (c *Client) ReadVersion(ctx context.Context, username, p, version string) (io.ReadCloser, error) { +func (c *Client) ReadVersion(ctx context.Context, uid, gid, p, version string) (io.ReadCloser, error) { basename := path.Base(p) versionFile := path.Join(path.Dir(p), versionPrefix+basename, version) - return c.Read(ctx, username, versionFile) + return c.Read(ctx, uid, gid, versionFile) } func parseRecycleList(raw string) ([]*DeletedEntry, error) { diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index d615357351..6973e2db57 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -89,6 +89,9 @@ type config struct { // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. UseKeytab bool `mapstructure:"use_keytab" docs:"false"` + + // GatewaySvc stores the endpoint at which the GRPC gateway is exposed. + GatewaySvc string `mapstructure:"gatewaysvc" docs:"0.0.0.0:19000"` } func parseConfig(m map[string]interface{}) (*config, error) { diff --git a/pkg/storage/fs/eoshome/eoshome.go b/pkg/storage/fs/eoshome/eoshome.go index b0c71a0879..aae0aeb03c 100644 --- a/pkg/storage/fs/eoshome/eoshome.go +++ b/pkg/storage/fs/eoshome/eoshome.go @@ -95,6 +95,9 @@ type config struct { // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. UseKeytab bool `mapstructure:"use_keytab" docs:"false"` + + // GatewaySvc stores the endpoint at which the GRPC gateway is exposed. + GatewaySvc string `mapstructure:"gatewaysvc" docs:"0.0.0.0:19000"` } func parseConfig(m map[string]interface{}) (*config, error) { diff --git a/pkg/storage/utils/acl/acl.go b/pkg/storage/utils/acl/acl.go index d9150e3aca..0c244f4273 100644 --- a/pkg/storage/utils/acl/acl.go +++ b/pkg/storage/utils/acl/acl.go @@ -20,6 +20,7 @@ package acl import ( "errors" + "fmt" "strings" ) @@ -131,6 +132,11 @@ func ParseEntry(singleSysACL string) (*Entry, error) { }, nil } +// CitrineSerialize serializes an ACL entry for citrine EOS ACLs +func (a *Entry) CitrineSerialize() string { + return fmt.Sprintf("%s:%s=%s", a.Type, a.Qualifier, a.Permissions) +} + func getShortType(aclType string) string { switch aclType[:1] { case TypeUser: diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 956bdcb78f..80bd11001a 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -25,26 +25,27 @@ import ( "io" "net/url" "os" - gouser "os/user" "path" "regexp" "strconv" "strings" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/utils/acl" "github.com/cs3org/reva/pkg/storage/utils/grants" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/errtypes" ) const ( @@ -119,6 +120,9 @@ type Config struct { // EnableHome enables the creation of home directories. EnableHome bool `mapstructure:"enable_home"` + + // GatewaySvc stores the endpoint at which the GRPC gateway is exposed. + GatewaySvc string `mapstructure:"gatewaysvc"` } func (c *Config) init() { @@ -160,11 +164,15 @@ func (c *Config) init() { if c.UserLayout == "" { c.UserLayout = "{{.Username}}" // TODO set better layout } + + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } type eosfs struct { - c *eosclient.Client - conf *Config + c *eosclient.Client + conf *Config + singleUserUID string + singleUserGID string } // NewEOSFS returns a storage.FS interface implementation that connects to an @@ -324,7 +332,12 @@ func (fs *eosfs) getPath(ctx context.Context, u *userpb.User, id *provider.Resou return "", fmt.Errorf("error converting string to int for eos fileid: %s", id.OpaqueId) } - eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fid) + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return "", err + } + + eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, uid, gid, fid) if err != nil { return "", errors.Wrap(err, "eos: error getting file info by inode") } @@ -359,13 +372,17 @@ func (fs *eosfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (stri return "", errors.Wrap(err, "eos: error parsing fileid string") } - eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fileID) + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return "", err + } + + eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, uid, gid, fileID) if err != nil { return "", errors.Wrap(err, "eos: error getting file info by inode") } - fi := fs.convertToResourceInfo(ctx, eosFileInfo) - return fi.Path, nil + return fs.unwrap(ctx, eosFileInfo.File), nil } func (fs *eosfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { @@ -389,12 +406,22 @@ func (fs *eosfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provi fn := fs.wrap(ctx, p) - eosACL, err := fs.getEosACL(g) + eosACL, err := fs.getEosACL(ctx, g) + if err != nil { + return err + } + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) if err != nil { return err } - err = fs.c.AddACL(ctx, u.Username, fn, eosACL) + rootUID, rootGID, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return err + } + + err = fs.c.AddACL(ctx, uid, gid, rootUID, rootGID, fn, eosACL) if err != nil { return errors.Wrap(err, "eos: error adding acl") } @@ -402,7 +429,7 @@ func (fs *eosfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provi return nil } -func (fs *eosfs) getEosACL(g *provider.Grant) (*acl.Entry, error) { +func (fs *eosfs) getEosACL(ctx context.Context, g *provider.Grant) (*acl.Entry, error) { permissions, err := grants.GetACLPerm(g.Permissions) if err != nil { return nil, err @@ -411,8 +438,19 @@ func (fs *eosfs) getEosACL(g *provider.Grant) (*acl.Entry, error) { if err != nil { return nil, err } + qualifier := g.Grantee.Id.OpaqueId + + // since EOS Citrine ACLs are stored with uid, we need to convert username to + // uid only for users. + if t == acl.TypeUser { + qualifier, _, err = fs.getUIDGateway(ctx, g.Grantee.Id) + if err != nil { + return nil, err + } + } + eosACL := &acl.Entry{ - Qualifier: g.Grantee.Id.OpaqueId, + Qualifier: qualifier, Permissions: permissions, Type: t, } @@ -429,45 +467,49 @@ func (fs *eosfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *pr if err != nil { return err } + recipient := g.Grantee.Id.OpaqueId - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") + // since EOS Citrine ACLs are stored with uid, we need to convert username to uid + if eosACLType == acl.TypeUser { + recipient, _, err = fs.getUIDGateway(ctx, g.Grantee.Id) + if err != nil { + return err + } } - fn := fs.wrap(ctx, p) - - err = fs.c.RemoveACL(ctx, u.Username, fn, eosACLType, g.Grantee.Id.OpaqueId) - if err != nil { - return errors.Wrap(err, "eos: error removing acl") + eosACL := &acl.Entry{ + Qualifier: recipient, + Type: eosACLType, } - return nil -} -func (fs *eosfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - u, err := getUser(ctx) + p, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "eos: no user in ctx") + return errors.Wrap(err, "eos: error resolving reference") } - eosACL, err := fs.getEosACL(g) + fn := fs.wrap(ctx, p) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) if err != nil { - return errors.Wrap(err, "eos: error mapping acl") + return err } - p, err := fs.resolve(ctx, u, ref) + rootUID, rootGID, err := fs.getRootUIDAndGID(ctx) if err != nil { - return errors.Wrap(err, "eos: error resolving reference") + return err } - fn := fs.wrap(ctx, p) - err = fs.c.AddACL(ctx, u.Username, fn, eosACL) + err = fs.c.RemoveACL(ctx, uid, gid, rootUID, rootGID, fn, eosACL) if err != nil { - return errors.Wrap(err, "eos: error updating acl") + return errors.Wrap(err, "eos: error removing acl") } return nil } +func (fs *eosfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return fs.AddGrant(ctx, ref, g) +} + func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { u, err := getUser(ctx) if err != nil { @@ -480,15 +522,29 @@ func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*pr } fn := fs.wrap(ctx, p) - acls, err := fs.c.ListACLs(ctx, u.Username, fn) + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + + acls, err := fs.c.ListACLs(ctx, uid, gid, fn) if err != nil { return nil, err } grantList := []*provider.Grant{} for _, a := range acls { + // EOS Citrine ACLs are stored with uid. + // This needs to be resolved to the user opaque ID. + qualifier := &userpb.UserId{OpaqueId: a.Qualifier} + if a.Type == acl.TypeUser { + qualifier, err = fs.getUserIDGateway(ctx, a.Qualifier) + if err != nil { + return nil, err + } + } grantee := &provider.Grantee{ - Id: &userpb.UserId{OpaqueId: a.Qualifier}, + Id: qualifier, Type: grants.GetGranteeType(a.Type), } grantList = append(grantList, &provider.Grant{ @@ -523,7 +579,12 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []st fn := fs.wrap(ctx, p) - eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + + eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, uid, gid, fn) if err != nil { return nil, err } @@ -539,7 +600,13 @@ func (fs *eosfs) getMDShareFolder(ctx context.Context, p string, mdKeys []string } fn := fs.wrapShadow(ctx, p) - eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + + eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, uid, gid, fn) if err != nil { return nil, err } @@ -585,9 +652,14 @@ func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*p return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + fn := fs.wrap(ctx, p) - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + eosFileInfos, err := fs.c.List(ctx, uid, gid, fn) if err != nil { return nil, errors.Wrap(err, "eos: error listing") } @@ -635,11 +707,16 @@ func (fs *eosfs) listHome(ctx context.Context, home string) ([]*provider.Resourc return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + fns := []string{fs.wrap(ctx, home), fs.wrapShadow(ctx, home)} finfos := []*provider.ResourceInfo{} for _, fn := range fns { - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + eosFileInfos, err := fs.c.List(ctx, uid, gid, fn) if err != nil { return nil, errors.Wrap(err, "eos: error listing") } @@ -666,9 +743,14 @@ func (fs *eosfs) listShareFolderRoot(ctx context.Context, p string) (finfos []*p return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + fn := fs.wrapShadow(ctx, p) - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + eosFileInfos, err := fs.c.List(ctx, uid, gid, fn) if err != nil { return nil, errors.Wrap(err, "eos: error listing") } @@ -695,7 +777,12 @@ func (fs *eosfs) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, errors.Wrap(err, "eos: no user in ctx") } - qi, err := fs.c.GetQuota(ctx, u.Username, fs.conf.Namespace) + rootUID, rootGID, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return 0, 0, err + } + + qi, err := fs.c.GetQuota(ctx, u.Username, rootUID, rootGID, fs.conf.Namespace) if err != nil { err := errors.Wrap(err, "eosfs: error getting quota") return 0, 0, err @@ -735,7 +822,11 @@ func (fs *eosfs) createShadowHome(ctx context.Context) error { } home := fs.wrapShadow(ctx, "/") - _, err = fs.c.GetFileInfoByPath(ctx, "root", home) + uid, gid, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return nil + } + _, err = fs.c.GetFileInfoByPath(ctx, uid, gid, home) if err == nil { // home already exists return nil } @@ -745,13 +836,13 @@ func (fs *eosfs) createShadowHome(ctx context.Context) error { return errors.Wrap(err, "eos: error verifying if user home directory exists") } - err = fs.createUserDir(ctx, u.Username, home) + err = fs.createUserDir(ctx, u, home) if err != nil { return err } shadowFolders := []string{fs.conf.ShareFolder} for _, sf := range shadowFolders { - err = fs.createUserDir(ctx, u.Username, path.Join(home, sf)) + err = fs.createUserDir(ctx, u, path.Join(home, sf)) if err != nil { return err } @@ -767,7 +858,11 @@ func (fs *eosfs) createNominalHome(ctx context.Context) error { } home := fs.wrap(ctx, "/") - _, err = fs.c.GetFileInfoByPath(ctx, "root", home) + uid, gid, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return nil + } + _, err = fs.c.GetFileInfoByPath(ctx, uid, gid, home) if err == nil { // home already exists return nil } @@ -777,7 +872,7 @@ func (fs *eosfs) createNominalHome(ctx context.Context) error { return errors.Wrap(err, "eos: error verifying if user home directory exists") } - err = fs.createUserDir(ctx, u.Username, home) + err = fs.createUserDir(ctx, u, home) return err } @@ -797,19 +892,29 @@ func (fs *eosfs) CreateHome(ctx context.Context) error { return nil } -func (fs *eosfs) createUserDir(ctx context.Context, username string, path string) error { - err := fs.c.CreateDir(ctx, "root", path) +func (fs *eosfs) createUserDir(ctx context.Context, u *userpb.User, path string) error { + uid, gid, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return nil + } + + chownUID, chownGID, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + + err = fs.c.CreateDir(ctx, uid, gid, path) if err != nil { // EOS will return success on mkdir over an existing directory. return errors.Wrap(err, "eos: error creating dir") } - err = fs.c.Chown(ctx, "root", username, path) + err = fs.c.Chown(ctx, uid, gid, chownUID, chownGID, path) if err != nil { return errors.Wrap(err, "eos: error chowning directory") } - err = fs.c.Chmod(ctx, "root", "2770", path) + err = fs.c.Chmod(ctx, uid, gid, "2770", path) if err != nil { return errors.Wrap(err, "eos: error chmoding directory") } @@ -838,7 +943,7 @@ func (fs *eosfs) createUserDir(ctx context.Context, username string, path string } for _, attr := range attrs { - err = fs.c.SetAttr(ctx, "root", attr, true, path) + err = fs.c.SetAttr(ctx, uid, gid, attr, true, path) if err != nil { return errors.Wrap(err, "eos: error setting attribute") } @@ -853,6 +958,11 @@ func (fs *eosfs) CreateDir(ctx context.Context, p string) error { return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + log.Info().Msgf("eos: createdir: path=%s", p) if fs.isShareFolder(ctx, p) { @@ -860,7 +970,7 @@ func (fs *eosfs) CreateDir(ctx context.Context, p string) error { } fn := fs.wrap(ctx, p) - return fs.c.CreateDir(ctx, u.Username, fn) + return fs.c.CreateDir(ctx, uid, gid, fn) } func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { @@ -877,7 +987,11 @@ func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.U // Current mechanism is: touch to hidden dir, set xattr, rename. dir, base := path.Split(fn) tmp := path.Join(dir, fmt.Sprintf(".sys.reva#.%s", base)) - if err := fs.c.CreateDir(ctx, "root", tmp); err != nil { + uid, gid, err := fs.getRootUIDAndGID(ctx) + if err != nil { + return nil + } + if err := fs.c.CreateDir(ctx, uid, gid, tmp); err != nil { err = errors.Wrapf(err, "eos: error creating temporary ref file") return err } @@ -889,13 +1003,13 @@ func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.U Val: targetURI.String(), } - if err := fs.c.SetAttr(ctx, "root", attr, false, tmp); err != nil { + if err := fs.c.SetAttr(ctx, uid, gid, attr, false, tmp); err != nil { err = errors.Wrapf(err, "eos: error setting reva.ref attr on file: %q", tmp) return err } // rename to have the file visible in user space. - if err := fs.c.Rename(ctx, "root", tmp, fn); err != nil { + if err := fs.c.Rename(ctx, uid, gid, tmp, fn); err != nil { err = errors.Wrapf(err, "eos: error renaming from: %q to %q", tmp, fn) return err } @@ -909,6 +1023,11 @@ func (fs *eosfs) Delete(ctx context.Context, ref *provider.Reference) error { return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + p, err := fs.resolve(ctx, u, ref) if err != nil { return errors.Wrap(err, "eos: error resolving reference") @@ -920,7 +1039,7 @@ func (fs *eosfs) Delete(ctx context.Context, ref *provider.Reference) error { fn := fs.wrap(ctx, p) - return fs.c.Remove(ctx, u.Username, fn) + return fs.c.Remove(ctx, uid, gid, fn) } func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { @@ -933,8 +1052,14 @@ func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { if err != nil { return errors.Wrap(err, "eos: no user in ctx") } + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + fn := fs.wrapShadow(ctx, p) - return fs.c.Remove(ctx, u.Username, fn) + return fs.c.Remove(ctx, uid, gid, fn) } panic("eos: shadow delete of share folder that is neither root nor child. path=" + p) @@ -946,6 +1071,11 @@ func (fs *eosfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) e return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + oldPath, err := fs.resolve(ctx, u, oldRef) if err != nil { return errors.Wrap(err, "eos: error resolving reference") @@ -962,7 +1092,7 @@ func (fs *eosfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) e oldFn := fs.wrap(ctx, oldPath) newFn := fs.wrap(ctx, newPath) - return fs.c.Rename(ctx, u.Username, oldFn, newFn) + return fs.c.Rename(ctx, uid, gid, oldFn, newFn) } func (fs *eosfs) moveShadow(ctx context.Context, oldPath, newPath string) error { @@ -971,6 +1101,11 @@ func (fs *eosfs) moveShadow(ctx context.Context, oldPath, newPath string) error return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + if fs.isShareFolderRoot(ctx, oldPath) || fs.isShareFolderRoot(ctx, newPath) { return errtypes.PermissionDenied("eos: cannot move/rename the virtual share folder") } @@ -985,7 +1120,7 @@ func (fs *eosfs) moveShadow(ctx context.Context, oldPath, newPath string) error oldfn := fs.wrapShadow(ctx, oldPath) newfn := fs.wrapShadow(ctx, newPath) - return fs.c.Rename(ctx, u.Username, oldfn, newfn) + return fs.c.Rename(ctx, uid, gid, oldfn, newfn) } func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { @@ -994,6 +1129,11 @@ func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.Read return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + p, err := fs.resolve(ctx, u, ref) if err != nil { return nil, errors.Wrap(err, "eos: error resolving reference") @@ -1005,7 +1145,7 @@ func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.Read fn := fs.wrap(ctx, p) - return fs.c.Read(ctx, u.Username, fn) + return fs.c.Read(ctx, uid, gid, fn) } func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { @@ -1014,6 +1154,11 @@ func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([] return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + p, err := fs.resolve(ctx, u, ref) if err != nil { return nil, errors.Wrap(err, "eos: error resolving reference") @@ -1025,7 +1170,7 @@ func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([] fn := fs.wrap(ctx, p) - eosRevisions, err := fs.c.ListVersions(ctx, u.Username, fn) + eosRevisions, err := fs.c.ListVersions(ctx, uid, gid, fn) if err != nil { return nil, errors.Wrap(err, "eos: error listing versions") } @@ -1043,6 +1188,11 @@ func (fs *eosfs) DownloadRevision(ctx context.Context, ref *provider.Reference, return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + p, err := fs.resolve(ctx, u, ref) if err != nil { return nil, errors.Wrap(err, "eos: error resolving reference") @@ -1055,7 +1205,7 @@ func (fs *eosfs) DownloadRevision(ctx context.Context, ref *provider.Reference, fn := fs.wrap(ctx, p) fn = fs.wrap(ctx, fn) - return fs.c.ReadVersion(ctx, u.Username, fn, revisionKey) + return fs.c.ReadVersion(ctx, uid, gid, fn, revisionKey) } func (fs *eosfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { @@ -1064,6 +1214,11 @@ func (fs *eosfs) RestoreRevision(ctx context.Context, ref *provider.Reference, r return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + p, err := fs.resolve(ctx, u, ref) if err != nil { return errors.Wrap(err, "eos: error resolving reference") @@ -1075,7 +1230,7 @@ func (fs *eosfs) RestoreRevision(ctx context.Context, ref *provider.Reference, r fn := fs.wrap(ctx, p) - return fs.c.RollbackToVersion(ctx, u.Username, fn, revisionKey) + return fs.c.RollbackToVersion(ctx, uid, gid, fn, revisionKey) } func (fs *eosfs) PurgeRecycleItem(ctx context.Context, key string) error { @@ -1083,7 +1238,13 @@ func (fs *eosfs) PurgeRecycleItem(ctx context.Context, key string) error { if err != nil { return errors.Wrap(err, "storage_eos: no user in ctx") } - return fs.c.RestoreDeletedEntry(ctx, u.Username, key) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + + return fs.c.RestoreDeletedEntry(ctx, uid, gid, key) } func (fs *eosfs) EmptyRecycle(ctx context.Context) error { @@ -1091,7 +1252,13 @@ func (fs *eosfs) EmptyRecycle(ctx context.Context) error { if err != nil { return errors.Wrap(err, "eos: no user in ctx") } - return fs.c.PurgeDeletedEntries(ctx, u.Username) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + + return fs.c.PurgeDeletedEntries(ctx, uid, gid) } func (fs *eosfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { @@ -1099,7 +1266,13 @@ func (fs *eosfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, erro if err != nil { return nil, errors.Wrap(err, "eos: no user in ctx") } - eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, u.Username) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return nil, err + } + + eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, uid, gid) if err != nil { return nil, errors.Wrap(err, "eos: error listing deleted entries") } @@ -1123,7 +1296,13 @@ func (fs *eosfs) RestoreRecycleItem(ctx context.Context, key string) error { if err != nil { return errors.Wrap(err, "eos: no user in ctx") } - return fs.c.RestoreDeletedEntry(ctx, u.Username, key) + + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } + + return fs.c.RestoreDeletedEntry(ctx, uid, gid, key) } func (fs *eosfs) convertToRecycleItem(ctx context.Context, eosDeletedItem *eosclient.DeletedEntry) *provider.RecycleItem { @@ -1176,17 +1355,17 @@ func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) * size = eosFileInfo.TreeSize } - username, err := getUsername(eosFileInfo.UID) + username, err := fs.getUserIDGateway(ctx, strconv.FormatUint(eosFileInfo.UID, 10)) if err != nil { log := appctx.GetLogger(ctx) log.Warn().Uint64("uid", eosFileInfo.UID).Msg("could not lookup userid, leaving empty") - username = "" // TODO(labkode): should we abort here? + username = &userpb.UserId{} } info := &provider.ResourceInfo{ Id: &provider.ResourceId{OpaqueId: fmt.Sprintf("%d", eosFileInfo.Inode)}, Path: path, - Owner: &userpb.UserId{OpaqueId: username}, + Owner: username, Etag: fmt.Sprintf("\"%s\"", strings.Trim(eosFileInfo.ETag, "\"")), MimeType: mime.Detect(eosFileInfo.IsDir, path), Size: size, @@ -1216,13 +1395,82 @@ func getResourceType(isDir bool) provider.ResourceType { return provider.ResourceType_RESOURCE_TYPE_FILE } -func getUsername(uid uint64) (string, error) { - s := strconv.FormatUint(uid, 10) - user, err := gouser.LookupId(s) +func (fs *eosfs) extractUIDAndGID(u *userpb.User) (string, string, error) { + var uid, gid string + if u.Opaque != nil && u.Opaque.Map != nil { + if uidObj, ok := u.Opaque.Map["uid"]; ok { + if uidObj.Decoder == "plain" { + uid = string(uidObj.Value) + } + } + if gidObj, ok := u.Opaque.Map["gid"]; ok { + if gidObj.Decoder == "plain" { + gid = string(gidObj.Value) + } + } + } + return uid, gid, nil +} + +func (fs *eosfs) getUIDGateway(ctx context.Context, u *userpb.UserId) (string, string, error) { + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) if err != nil { - return "", err + return "", "", errors.Wrap(err, "eos: error getting gateway grpc client") + } + getUserResp, err := client.GetUser(ctx, &userpb.GetUserRequest{ + UserId: u, + }) + if err != nil { + return "", "", errors.Wrap(err, "eos: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return "", "", errors.Wrap(err, "eos: grpc get user failed") + } + return fs.extractUIDAndGID(getUserResp.User) +} + +func (fs *eosfs) getUserIDGateway(ctx context.Context, uid string) (*userpb.UserId, error) { + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "eos: error getting gateway grpc client") + } + getUserResp, err := client.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{ + Claim: "uid", + Value: uid, + }) + if err != nil { + return nil, errors.Wrap(err, "eos: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "eos: grpc get user failed") + } + return getUserResp.User.Id, nil +} + +func (fs *eosfs) getUserUIDAndGID(ctx context.Context, u *userpb.User) (string, string, error) { + if fs.conf.ForceSingleUserMode { + if fs.singleUserUID != "" && fs.singleUserGID != "" { + return fs.singleUserUID, fs.singleUserGID, nil + } + uid, gid, err := fs.getUIDGateway(ctx, &userpb.UserId{OpaqueId: fs.conf.SingleUsername}) + fs.singleUserUID = uid + fs.singleUserGID = gid + return fs.singleUserUID, fs.singleUserGID, err + } + return fs.extractUIDAndGID(u) +} + +func (fs *eosfs) getRootUIDAndGID(ctx context.Context) (string, string, error) { + if fs.conf.ForceSingleUserMode { + if fs.singleUserUID != "" && fs.singleUserGID != "" { + return fs.singleUserUID, fs.singleUserGID, nil + } + uid, gid, err := fs.getUIDGateway(ctx, &userpb.UserId{OpaqueId: fs.conf.SingleUsername}) + fs.singleUserUID = uid + fs.singleUserGID = gid + return fs.singleUserUID, fs.singleUserGID, err } - return user.Username, nil + return "0", "0", nil } type eosSysMetadata struct { diff --git a/pkg/storage/utils/eosfs/upload.go b/pkg/storage/utils/eosfs/upload.go index 89028c546a..ce24e60b3f 100644 --- a/pkg/storage/utils/eosfs/upload.go +++ b/pkg/storage/utils/eosfs/upload.go @@ -43,6 +43,10 @@ func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadC if err != nil { return errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, u) + if err != nil { + return err + } p, err := fs.resolve(ctx, u, ref) if err != nil { @@ -55,7 +59,7 @@ func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadC fn := fs.wrap(ctx, p) - return fs.c.Write(ctx, u.Username, fn, r) + return fs.c.Write(ctx, uid, gid, fn, r) } // InitiateUpload returns an upload id that can be used for uploads with tus @@ -133,9 +137,16 @@ func (fs *eosfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd if err != nil { return nil, errors.Wrap(err, "eos: no user in ctx") } + uid, gid, err := fs.getUserUIDAndGID(ctx, user) + if err != nil { + return nil, err + } + info.Storage = map[string]string{ "Type": "EOSStore", "Username": user.Username, + "UID": uid, + "GID": gid, } // Create binary file with no content @@ -292,7 +303,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { // eos creates revisions internally //} - err := upload.fs.c.WriteFile(ctx, upload.info.Storage["Username"], np, upload.binPath) + err := upload.fs.c.WriteFile(ctx, upload.info.Storage["UID"], upload.info.Storage["GID"], np, upload.binPath) // only delete the upload if it was successfully written to eos if err == nil { diff --git a/pkg/user/manager/demo/demo.go b/pkg/user/manager/demo/demo.go index 04440892ac..de61836dc2 100644 --- a/pkg/user/manager/demo/demo.go +++ b/pkg/user/manager/demo/demo.go @@ -20,9 +20,11 @@ package demo import ( "context" + "errors" "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" @@ -49,6 +51,33 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User return nil, errtypes.NotFound(uid.OpaqueId) } +func (m *manager) GetUserByClaim(ctx context.Context, claim, value string) (*userpb.User, error) { + for _, u := range m.catalog { + if userClaim, err := extractClaim(u, claim); err == nil && value == userClaim { + return u, nil + } + } + return nil, errtypes.NotFound(value) +} + +func extractClaim(u *userpb.User, claim string) (string, error) { + switch claim { + case "mail": + return u.Mail, nil + case "username": + return u.Username, nil + case "uid": + if u.Opaque != nil && u.Opaque.Map != nil { + if uidObj, ok := u.Opaque.Map["uid"]; ok { + if uidObj.Decoder == "plain" { + return string(uidObj.Value), nil + } + } + } + } + return "", errors.New("demo: invalid field") +} + // TODO(jfd) search Opaque? compare sub? func userContains(u *userpb.User, query string) bool { return strings.Contains(u.Username, query) || strings.Contains(u.DisplayName, query) || strings.Contains(u.Mail, query) @@ -97,6 +126,18 @@ func getUsers() map[string]*userpb.User { Groups: []string{"sailing-lovers", "violin-haters", "physics-lovers"}, Mail: "einstein@example.org", DisplayName: "Albert Einstein", + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("123"), + }, + "gid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("987"), + }, + }, + }, }, "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c": &userpb.User{ Id: &userpb.UserId{ @@ -107,6 +148,18 @@ func getUsers() map[string]*userpb.User { Groups: []string{"radium-lovers", "polonium-lovers", "physics-lovers"}, Mail: "marie@example.org", DisplayName: "Marie Curie", + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("456"), + }, + "gid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("987"), + }, + }, + }, }, "932b4540-8d16-481e-8ef4-588e4b6b151c": &userpb.User{ Id: &userpb.UserId{ diff --git a/pkg/user/manager/demo/demo_test.go b/pkg/user/manager/demo/demo_test.go index 440b8005e9..f94963109c 100644 --- a/pkg/user/manager/demo/demo_test.go +++ b/pkg/user/manager/demo/demo_test.go @@ -24,6 +24,7 @@ import ( "testing" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/errtypes" ) @@ -41,6 +42,12 @@ func TestUserManager(t *testing.T) { Groups: []string{"sailing-lovers", "violin-haters", "physics-lovers"}, Mail: "einstein@example.org", DisplayName: "Albert Einstein", + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{Decoder: "plain", Value: []byte("123")}, + "gid": &types.OpaqueEntry{Decoder: "plain", Value: []byte("987")}, + }, + }, } uidFake := &userpb.UserId{Idp: "nonesense", OpaqueId: "fakeUser"} groupsEinstein := []string{"sailing-lovers", "violin-haters", "physics-lovers"} @@ -48,20 +55,39 @@ func TestUserManager(t *testing.T) { // positive test GetUserGroups resGroups, _ := manager.GetUserGroups(ctx, uidEinstein) if !reflect.DeepEqual(resGroups, groupsEinstein) { - t.Fatalf("groups differ: expected=%v got=%v", resGroups, groupsEinstein) + t.Fatalf("groups differ: expected=%v got=%v", groupsEinstein, resGroups) } // negative test GetUserGroups expectedErr := errtypes.NotFound(uidFake.OpaqueId) _, err := manager.GetUserGroups(ctx, uidFake) if !reflect.DeepEqual(err, expectedErr) { - t.Fatalf("user not found error differ: expected='%v' got='%v'", expectedErr, err) + t.Fatalf("user not found error differs: expected='%v' got='%v'", expectedErr, err) + } + + // positive test GetUserByClaim by uid + resUserByUID, _ := manager.GetUserByClaim(ctx, "uid", "123") + if !reflect.DeepEqual(resUserByUID, userEinstein) { + t.Fatalf("user differs: expected=%v got=%v", userEinstein, resUserByUID) + } + + // negative test GetUserByClaim by uid + expectedErr = errtypes.NotFound("789") + _, err = manager.GetUserByClaim(ctx, "uid", "789") + if !reflect.DeepEqual(err, expectedErr) { + t.Fatalf("user not found error differs: expected='%v' got='%v'", expectedErr, err) + } + + // positive test GetUserByClaim by mail + resUserByEmail, _ := manager.GetUserByClaim(ctx, "mail", "einstein@example.org") + if !reflect.DeepEqual(resUserByEmail, userEinstein) { + t.Fatalf("user differs: expected=%v got=%v", userEinstein, resUserByEmail) } // test FindUsers resUser, _ := manager.FindUsers(ctx, "einstein") if !reflect.DeepEqual(resUser, []*userpb.User{userEinstein}) { - t.Fatalf("user differ: expected=%v got=%v", []*userpb.User{userEinstein}, resUser) + t.Fatalf("user differs: expected=%v got=%v", []*userpb.User{userEinstein}, resUser) } // negative test FindUsers @@ -86,9 +112,9 @@ func TestUserManager(t *testing.T) { expectedErr = errtypes.NotFound(uidFake.OpaqueId) resInGroup, err = manager.IsInGroup(ctx, uidFake, "physics-lovers") if !reflect.DeepEqual(err, expectedErr) { - t.Fatalf("user not in group error differ: expected='%v' got='%v'", expectedErr, err) + t.Fatalf("user not in group error differs: expected='%v' got='%v'", expectedErr, err) } if resInGroup { - t.Fatalf("user not in group bool differ: expected='%v' got='%v'", false, resInGroup) + t.Fatalf("user not in group bool differs: expected='%v' got='%v'", false, resInGroup) } } diff --git a/pkg/user/manager/json/json.go b/pkg/user/manager/json/json.go index 82cc02eb6a..7202082e0a 100644 --- a/pkg/user/manager/json/json.go +++ b/pkg/user/manager/json/json.go @@ -95,6 +95,33 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User return nil, errtypes.NotFound(uid.OpaqueId) } +func (m *manager) GetUserByClaim(ctx context.Context, claim, value string) (*userpb.User, error) { + for _, u := range m.users { + if userClaim, err := extractClaim(u, claim); err == nil && value == userClaim { + return u, nil + } + } + return nil, errtypes.NotFound(value) +} + +func extractClaim(u *userpb.User, claim string) (string, error) { + switch claim { + case "mail": + return u.Mail, nil + case "username": + return u.Username, nil + case "uid": + if u.Opaque != nil && u.Opaque.Map != nil { + if uidObj, ok := u.Opaque.Map["uid"]; ok { + if uidObj.Decoder == "plain" { + return string(uidObj.Value), nil + } + } + } + } + return "", errors.New("json: invalid field") +} + // TODO(jfd) search Opaque? compare sub? func userContains(u *userpb.User, query string) bool { return strings.Contains(u.Username, query) || strings.Contains(u.DisplayName, query) || strings.Contains(u.Mail, query) || strings.Contains(u.Id.OpaqueId, query) diff --git a/pkg/user/manager/json/json_test.go b/pkg/user/manager/json/json_test.go index 0eabc2fb04..52ced0df2a 100644 --- a/pkg/user/manager/json/json_test.go +++ b/pkg/user/manager/json/json_test.go @@ -89,12 +89,19 @@ func TestUserManager(t *testing.T) { manager, _ := New(input) // setup test data - userEinstein := &userpb.UserId{Idp: "localhost", OpaqueId: "einstein"} + uidEinstein := &userpb.UserId{Idp: "localhost", OpaqueId: "einstein"} + userEinstein := &userpb.User{ + Id: uidEinstein, + Username: "einstein", + Groups: []string{"sailing-lovers", "violin-haters", "physics-lovers"}, + Mail: "einstein@example.org", + DisplayName: "Albert Einstein", + } userFake := &userpb.UserId{Idp: "localhost", OpaqueId: "fakeUser"} groupsEinstein := []string{"sailing-lovers", "violin-haters", "physics-lovers"} // positive test GetUserGroups - resGroups, _ := manager.GetUserGroups(ctx, userEinstein) + resGroups, _ := manager.GetUserGroups(ctx, uidEinstein) if !reflect.DeepEqual(resGroups, groupsEinstein) { t.Fatalf("groups differ: expected=%v got=%v", resGroups, groupsEinstein) } @@ -106,6 +113,19 @@ func TestUserManager(t *testing.T) { t.Fatalf("user not found error differ: expected='%v' got='%v'", expectedErr, err) } + // positive test GetUserByClaim by mail + resUserByEmail, _ := manager.GetUserByClaim(ctx, "mail", "einstein@example.org") + if !reflect.DeepEqual(resUserByEmail, userEinstein) { + t.Fatalf("user differs: expected=%v got=%v", userEinstein, resUserByEmail) + } + + // negative test GetUserByClaim by mail + expectedErr = errtypes.NotFound("abc@example.com") + _, err = manager.GetUserByClaim(ctx, "mail", "abc@example.com") + if !reflect.DeepEqual(err, expectedErr) { + t.Fatalf("user not found error differs: expected='%v' got='%v'", expectedErr, err) + } + // test FindUsers resUser, _ := manager.FindUsers(ctx, "stein") if len(resUser) != 1 { @@ -116,13 +136,13 @@ func TestUserManager(t *testing.T) { } // positive test IsInGroup - resInGroup, _ := manager.IsInGroup(ctx, userEinstein, "physics-lovers") + resInGroup, _ := manager.IsInGroup(ctx, uidEinstein, "physics-lovers") if !resInGroup { t.Fatalf("user not in group: expected=%v got=%v", true, false) } // negative test IsInGroup with wrong group - resInGroup, _ = manager.IsInGroup(ctx, userEinstein, "notARealGroup") + resInGroup, _ = manager.IsInGroup(ctx, uidEinstein, "notARealGroup") if resInGroup { t.Fatalf("user not in group: expected=%v got=%v", true, false) } diff --git a/pkg/user/manager/ldap/ldap.go b/pkg/user/manager/ldap/ldap.go index 1907b0bc09..89473c8b27 100644 --- a/pkg/user/manager/ldap/ldap.go +++ b/pkg/user/manager/ldap/ldap.go @@ -179,6 +179,10 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User return u, nil } +func (m *manager) GetUserByClaim(ctx context.Context, field, claim string) (*userpb.User, error) { + return nil, errtypes.NotSupported("ldap: looking up user by specific field not supported") +} + func (m *manager) FindUsers(ctx context.Context, query string) ([]*userpb.User, error) { l, err := ldap.DialTLS("tcp", fmt.Sprintf("%s:%d", m.c.Hostname, m.c.Port), &tls.Config{InsecureSkipVerify: true}) if err != nil { diff --git a/pkg/user/manager/rest/cache.go b/pkg/user/manager/rest/cache.go index 1a639a043b..5b27064625 100644 --- a/pkg/user/manager/rest/cache.go +++ b/pkg/user/manager/rest/cache.go @@ -28,11 +28,12 @@ import ( ) const ( - userDetailsPrefix = "user:" - userGroupsPrefix = "groups:" + userDetailsPrefix = "user:" + userGroupsPrefix = "groups:" + userInternalIDPrefix = "internal:" ) -func initRedisPool(port string) *redis.Pool { +func initRedisPool(address, username, password string) *redis.Pool { return &redis.Pool{ MaxIdle: 50, @@ -40,7 +41,22 @@ func initRedisPool(port string) *redis.Pool { IdleTimeout: 240 * time.Second, Dial: func() (redis.Conn, error) { - c, err := redis.Dial("tcp", port) + var c redis.Conn + var err error + switch { + case username != "": + c, err = redis.Dial("tcp", address, + redis.DialUsername(username), + redis.DialPassword(password), + ) + case password != "": + c, err = redis.Dial("tcp", address, + redis.DialPassword(password), + ) + default: + c, err = redis.Dial("tcp", address) + } + if err != nil { return nil, err } @@ -54,68 +70,107 @@ func initRedisPool(port string) *redis.Pool { } } -func (m *manager) fetchCachedUserDetails(uid *userpb.UserId) (*userpb.User, error) { +func (m *manager) setVal(key, val string, expiration int) error { conn := m.redisPool.Get() defer conn.Close() if conn != nil { - user, err := redis.String(conn.Do("GET", userDetailsPrefix+uid.OpaqueId)) - if err != nil { - return nil, err - } - u := userpb.User{} - if err = json.Unmarshal([]byte(user), &u); err != nil { - return nil, err + if expiration != -1 { + if _, err := conn.Do("SET", key, val, "EX", expiration); err != nil { + return err + } + } else { + if _, err := conn.Do("SET", key, val); err != nil { + return err + } } - return &u, nil + return nil } - return nil, errors.New("rest: unable to get connection from redis pool") + return errors.New("rest: unable to get connection from redis pool") } -func (m *manager) cacheUserDetails(u *userpb.User) error { +func (m *manager) getVal(key string) (string, error) { conn := m.redisPool.Get() defer conn.Close() if conn != nil { - encodedUser, err := json.Marshal(&u) + val, err := redis.String(conn.Do("GET", key)) if err != nil { - return err + return "", err } - if _, err = conn.Do("SET", userDetailsPrefix+u.Id.OpaqueId, string(encodedUser)); err != nil { - return err - } - return nil + return val, nil } - return errors.New("rest: unable to get connection from redis pool") + return "", errors.New("rest: unable to get connection from redis pool") +} + +func (m *manager) fetchCachedInternalID(uid *userpb.UserId) (string, error) { + return m.getVal(userInternalIDPrefix + uid.OpaqueId) +} + +func (m *manager) cacheInternalID(uid *userpb.UserId, internalID string) error { + return m.setVal(userInternalIDPrefix+uid.OpaqueId, internalID, -1) +} + +func (m *manager) fetchCachedUserDetails(uid *userpb.UserId) (*userpb.User, error) { + user, err := m.getVal(userDetailsPrefix + uid.OpaqueId) + if err != nil { + return nil, err + } + + u := userpb.User{} + if err = json.Unmarshal([]byte(user), &u); err != nil { + return nil, err + } + return &u, nil +} + +func (m *manager) cacheUserDetails(u *userpb.User) error { + encodedUser, err := json.Marshal(&u) + if err != nil { + return err + } + if err = m.setVal(userDetailsPrefix+u.Id.OpaqueId, string(encodedUser), -1); err != nil { + return err + } + + uid, err := extractUID(u) + if err != nil { + return err + } + + if err = m.setVal("uid:"+uid, u.Id.OpaqueId, -1); err != nil { + return err + } + if err = m.setVal("mail:"+u.Mail, u.Id.OpaqueId, -1); err != nil { + return err + } + if err = m.setVal("username:"+u.Username, u.Id.OpaqueId, -1); err != nil { + return err + } + return nil +} + +func (m *manager) fetchCachedParam(field, claim string) (string, error) { + return m.getVal(field + ":" + claim) } func (m *manager) fetchCachedUserGroups(uid *userpb.UserId) ([]string, error) { - conn := m.redisPool.Get() - defer conn.Close() - if conn != nil { - groups, err := redis.String(conn.Do("GET", userGroupsPrefix+uid.OpaqueId)) - if err != nil { - return nil, err - } - g := []string{} - if err = json.Unmarshal([]byte(groups), &g); err != nil { - return nil, err - } - return g, nil + groups, err := m.getVal(userGroupsPrefix + uid.OpaqueId) + if err != nil { + return nil, err } - return nil, errors.New("rest: unable to get connection from redis pool") + g := []string{} + if err = json.Unmarshal([]byte(groups), &g); err != nil { + return nil, err + } + return g, nil } func (m *manager) cacheUserGroups(uid *userpb.UserId, groups []string) error { - conn := m.redisPool.Get() - defer conn.Close() - if conn != nil { - encodedGroups, err := json.Marshal(&groups) - if err != nil { - return err - } - if _, err = conn.Do("SET", userGroupsPrefix+uid.OpaqueId, string(encodedGroups), "EX", m.conf.UserGroupsCacheExpiration*60); err != nil { - return err - } - return nil + g, err := json.Marshal(&groups) + if err != nil { + return err } - return errors.New("rest: unable to get connection from redis pool") + if err = m.setVal(userGroupsPrefix+uid.OpaqueId, string(g), m.conf.UserGroupsCacheExpiration*60); err != nil { + return err + } + return nil } diff --git a/pkg/user/manager/rest/rest.go b/pkg/user/manager/rest/rest.go index 5f3568873b..7b9f92e5b3 100644 --- a/pkg/user/manager/rest/rest.go +++ b/pkg/user/manager/rest/rest.go @@ -32,11 +32,11 @@ import ( "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" - "github.com/gomodule/redigo/redis" "github.com/mitchellh/mapstructure" ) @@ -64,8 +64,12 @@ type OIDCToken struct { } type config struct { - // The port on which the redis server is running - Redis string `mapstructure:"redis" docs:":6379"` + // The address at which the redis server is running + RedisAddress string `mapstructure:"redis_address" docs:"localhost:6379"` + // The username for connecting to the redis server + RedisUsername string `mapstructure:"redis_username" docs:""` + // The password for connecting to the redis server + RedisPassword string `mapstructure:"redis_password" docs:""` // The time in minutes for which the groups to which a user belongs would be cached UserGroupsCacheExpiration int `mapstructure:"user_groups_cache_expiration" docs:"5"` // The OIDC Provider @@ -87,8 +91,8 @@ func (c *config) init() { if c.UserGroupsCacheExpiration == 0 { c.UserGroupsCacheExpiration = 5 } - if c.Redis == "" { - c.Redis = ":6379" + if c.RedisAddress == "" { + c.RedisAddress = ":6379" } if c.APIBaseURL == "" { c.APIBaseURL = "https://authorization-service-api-dev.web.cern.ch/api/v1.0" @@ -120,7 +124,7 @@ func New(m map[string]interface{}) (user.Manager, error) { } c.init() - redisPool := initRedisPool(c.Redis) + redisPool := initRedisPool(c.RedisAddress, c.RedisUsername, c.RedisPassword) return &manager{ conf: c, redisPool: redisPool, @@ -224,31 +228,87 @@ func (m *manager) sendAPIRequest(ctx context.Context, url string) ([]interface{} return responseData, nil } -func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User, error) { +func (m *manager) getUserByParam(ctx context.Context, param, val string) (map[string]interface{}, error) { + url := fmt.Sprintf("%s/Identity?filter=%s:%s&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid", + m.conf.APIBaseURL, param, val) + responseData, err := m.sendAPIRequest(ctx, url) + if err != nil { + return nil, err + } + userData, ok := responseData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("rest: error in type assertion") + } + return userData, nil +} - u, err := m.fetchCachedUserDetails(uid) +func (m *manager) getInternalUserID(ctx context.Context, uid *userpb.UserId) (string, error) { + + internalID, err := m.fetchCachedInternalID(uid) if err != nil { - url := fmt.Sprintf("%s/Identity/?filter=id:%s&field=upn&field=primaryAccountEmail&field=displayName", m.conf.APIBaseURL, uid.OpaqueId) - responseData, err := m.sendAPIRequest(ctx, url) + userData, err := m.getUserByParam(ctx, "upn", uid.OpaqueId) if err != nil { - return nil, err + return "", err } - - userData, ok := responseData[0].(map[string]interface{}) + id, ok := userData["id"].(string) if !ok { - return nil, errors.New("rest: error in type assertion") - } - u = &userpb.User{ - Id: uid, - Username: userData["upn"].(string), - Mail: userData["primaryAccountEmail"].(string), - DisplayName: userData["displayName"].(string), + return "", errors.New("rest: error in type assertion") } - if err = m.cacheUserDetails(u); err != nil { + if err = m.cacheInternalID(uid, id); err != nil { log := appctx.GetLogger(ctx) log.Error().Err(err).Msg("rest: error caching user details") } + return id, nil + } + return internalID, nil +} + +func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) *userpb.User { + userID := &userpb.UserId{ + OpaqueId: userData["upn"].(string), + Idp: m.conf.IDProvider, + } + u := &userpb.User{ + Id: userID, + Username: userData["upn"].(string), + Mail: userData["primaryAccountEmail"].(string), + DisplayName: userData["displayName"].(string), + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", userData["uid"])), + }, + "gid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", userData["gid"])), + }, + }, + }, + } + + if err := m.cacheUserDetails(u); err != nil { + log := appctx.GetLogger(ctx) + log.Error().Err(err).Msg("rest: error caching user details") + } + if err := m.cacheInternalID(userID, userData["id"].(string)); err != nil { + log := appctx.GetLogger(ctx) + log.Error().Err(err).Msg("rest: error caching user details") + } + return u + +} + +func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User, error) { + + u, err := m.fetchCachedUserDetails(uid) + if err != nil { + userData, err := m.getUserByParam(ctx, "upn", uid.OpaqueId) + if err != nil { + return nil, err + } + u = m.parseAndCacheUser(ctx, userData) } userGroups, err := m.GetUserGroups(ctx, uid) @@ -260,6 +320,39 @@ func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User return u, nil } +func (m *manager) GetUserByClaim(ctx context.Context, claim, value string) (*userpb.User, error) { + opaqueID, err := m.fetchCachedParam(claim, value) + if err == nil { + return m.GetUser(ctx, &userpb.UserId{OpaqueId: opaqueID}) + } + + switch claim { + case "mail": + claim = "primaryAccountEmail" + case "uid": + claim = "uid" + case "username": + claim = "upn" + default: + return nil, errors.New("rest: invalid field") + } + + userData, err := m.getUserByParam(ctx, claim, value) + if err != nil { + return nil, err + } + u := m.parseAndCacheUser(ctx, userData) + + userGroups, err := m.GetUserGroups(ctx, u.Id) + if err != nil { + return nil, err + } + u.Groups = userGroups + + return u, nil + +} + func (m *manager) findUsersByFilter(ctx context.Context, url string) ([]*userpb.User, error) { userData, err := m.sendAPIRequest(ctx, url) @@ -276,19 +369,26 @@ func (m *manager) findUsersByFilter(ctx context.Context, url string) ([]*userpb. } uid := &userpb.UserId{ - OpaqueId: usrInfo["id"].(string), + OpaqueId: usrInfo["upn"].(string), Idp: m.conf.IDProvider, } - userGroups, err := m.GetUserGroups(ctx, uid) - if err != nil { - return nil, err - } users = append(users, &userpb.User{ Id: uid, Username: usrInfo["upn"].(string), Mail: usrInfo["primaryAccountEmail"].(string), DisplayName: usrInfo["displayName"].(string), - Groups: userGroups, + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", usrInfo["uid"])), + }, + "gid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(fmt.Sprintf("%0.f", usrInfo["gid"])), + }, + }, + }, }) } @@ -310,7 +410,8 @@ func (m *manager) FindUsers(ctx context.Context, query string) ([]*userpb.User, users := []*userpb.User{} for _, f := range filters { - url := fmt.Sprintf("%s/Identity/?filter=%s:contains:%s&field=id&field=upn&field=primaryAccountEmail&field=displayName", m.conf.APIBaseURL, f, query) + url := fmt.Sprintf("%s/Identity/?filter=%s:contains:%s&field=id&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid", + m.conf.APIBaseURL, f, query) filteredUsers, err := m.findUsersByFilter(ctx, url) if err != nil { return nil, err @@ -327,7 +428,11 @@ func (m *manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]stri return groups, nil } - url := fmt.Sprintf("%s/Identity/%s/groups", m.conf.APIBaseURL, uid.OpaqueId) + internalID, err := m.getInternalUserID(ctx, uid) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s/Identity/%s/groups", m.conf.APIBaseURL, internalID) groupData, err := m.sendAPIRequest(ctx, url) if err != nil { return nil, err @@ -364,3 +469,14 @@ func (m *manager) IsInGroup(ctx context.Context, uid *userpb.UserId, group strin } return false, nil } + +func extractUID(u *userpb.User) (string, error) { + if u.Opaque != nil && u.Opaque.Map != nil { + if uidObj, ok := u.Opaque.Map["uid"]; ok { + if uidObj.Decoder == "plain" { + return string(uidObj.Value), nil + } + } + } + return "", errors.New("rest: could not retrieve UID from user") +} diff --git a/pkg/user/user.go b/pkg/user/user.go index c856f7b13a..dd22062f82 100644 --- a/pkg/user/user.go +++ b/pkg/user/user.go @@ -65,6 +65,7 @@ func ContextSetUserID(ctx context.Context, id *userpb.UserId) context.Context { // Manager is the interface to implement to manipulate users. type Manager interface { GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User, error) + GetUserByClaim(ctx context.Context, claim, value string) (*userpb.User, error) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]string, error) IsInGroup(ctx context.Context, uid *userpb.UserId, group string) (bool, error) FindUsers(ctx context.Context, query string) ([]*userpb.User, error)