From 983f0b3e30906c873cb5d7ed33e0f35a93896738 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 10 Feb 2021 15:52:54 +0100 Subject: [PATCH 1/2] eos: fix default location for xrdcopy binary --- changelog/unreleased/xrdcopy-default.md | 3 + .../packages/auth/manager/oidc/_index.md | 18 +++-- .../docs/config/packages/cbox/group/_index.md | 7 ++ .../config/packages/cbox/group/rest/_index.md | 74 +++++++++++++++++++ go.mod | 1 + pkg/eosclient/eosbinary/eosbinary.go | 4 +- pkg/eosclient/eosgrpc/eosgrpc.go | 4 +- pkg/storage/fs/ocis/ocis_test.go | 25 ++++--- pkg/storage/utils/eosfs/config.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 2 +- 10 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 changelog/unreleased/xrdcopy-default.md create mode 100644 docs/content/en/docs/config/packages/cbox/group/_index.md create mode 100644 docs/content/en/docs/config/packages/cbox/group/rest/_index.md diff --git a/changelog/unreleased/xrdcopy-default.md b/changelog/unreleased/xrdcopy-default.md new file mode 100644 index 0000000000..98ad4da3a9 --- /dev/null +++ b/changelog/unreleased/xrdcopy-default.md @@ -0,0 +1,3 @@ +Enhancement: Align default location for xrdcopy binary + +https://github.com/cs3org/reva/pull/1431 diff --git a/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md index 0df3479857..5e45799793 100644 --- a/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md +++ b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L50) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L53) {{< highlight toml >}} [auth.manager.oidc] insecure = false @@ -17,7 +17,7 @@ insecure = false {{% /dir %}} {{% dir name="issuer" type="string" default="" %}} -The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L51) +The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L54) {{< highlight toml >}} [auth.manager.oidc] issuer = "" @@ -25,7 +25,7 @@ issuer = "" {{% /dir %}} {{% dir name="id_claim" type="string" default="sub" %}} -The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L52) +The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L55) {{< highlight toml >}} [auth.manager.oidc] id_claim = "sub" @@ -33,7 +33,7 @@ id_claim = "sub" {{% /dir %}} {{% dir name="uid_claim" type="string" default="" %}} -The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L53) +The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L56) {{< highlight toml >}} [auth.manager.oidc] uid_claim = "" @@ -41,10 +41,18 @@ uid_claim = "" {{% /dir %}} {{% dir name="gid_claim" type="string" default="" %}} -The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L54) +The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L57) {{< highlight toml >}} [auth.manager.oidc] gid_claim = "" {{< /highlight >}} {{% /dir %}} +{{% dir name="gatewaysvc" type="string" default="" %}} +The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L58) +{{< highlight toml >}} +[auth.manager.oidc] +gatewaysvc = "" +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/cbox/group/_index.md b/docs/content/en/docs/config/packages/cbox/group/_index.md new file mode 100644 index 0000000000..55e54590a0 --- /dev/null +++ b/docs/content/en/docs/config/packages/cbox/group/_index.md @@ -0,0 +1,7 @@ +--- +title: "group" +linkTitle: "group" +weight: 10 +description: > + Configuration for the group service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/cbox/group/rest/_index.md b/docs/content/en/docs/config/packages/cbox/group/rest/_index.md new file mode 100644 index 0000000000..de5cbcec53 --- /dev/null +++ b/docs/content/en/docs/config/packages/cbox/group/rest/_index.md @@ -0,0 +1,74 @@ +--- +title: "rest" +linkTitle: "rest" +weight: 10 +description: > + Configuration for the rest service +--- + +# _struct: config_ + +{{% dir name="redis_address" type="string" default="localhost:6379" %}} +The address at which the redis server is running [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L69) +{{< highlight toml >}} +[cbox.group.rest] +redis_address = "localhost:6379" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="user_groups_cache_expiration" type="int" default=5 %}} +The time in minutes for which the members of a group would be cached [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L75) +{{< highlight toml >}} +[cbox.group.rest] +user_groups_cache_expiration = 5 +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="id_provider" type="string" default="http://cernbox.cern.ch" %}} +The OIDC Provider [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L77) +{{< highlight toml >}} +[cbox.group.rest] +id_provider = "http://cernbox.cern.ch" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="api_base_url" type="string" default="https://authorization-service-api-dev.web.cern.ch/api/v1.0" %}} +Base API Endpoint [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L79) +{{< highlight toml >}} +[cbox.group.rest] +api_base_url = "https://authorization-service-api-dev.web.cern.ch/api/v1.0" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="client_id" type="string" default="-" %}} +Client ID needed to authenticate [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L81) +{{< highlight toml >}} +[cbox.group.rest] +client_id = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="client_secret" type="string" default="-" %}} +Client Secret [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L83) +{{< highlight toml >}} +[cbox.group.rest] +client_secret = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="oidc_token_endpoint" type="string" default="https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" %}} +Endpoint to generate token to access the API [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L86) +{{< highlight toml >}} +[cbox.group.rest] +oidc_token_endpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="target_api" type="string" default="authorization-service-api" %}} +The target application for which token needs to be generated [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L88) +{{< highlight toml >}} +[cbox.group.rest] +target_api = "authorization-service-api" +{{< /highlight >}} +{{% /dir %}} + diff --git a/go.mod b/go.mod index 1ad1f7bb40..e9e4ecaada 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1 github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2 go.opencensus.io v0.22.6 + golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index e858974336..8db63bdcc2 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -102,7 +102,7 @@ type Options struct { EosBinary string // Location of the xrdcopy binary. - // Default is /usr/bin/xrdcopy. + // Default is /opt/eos/xrootd/bin/xrdcopy. XrdcopyBinary string // URL of the EOS MGM. @@ -131,7 +131,7 @@ func (opt *Options) init() { } if opt.XrdcopyBinary == "" { - opt.XrdcopyBinary = "/usr/bin/xrdcopy" + opt.XrdcopyBinary = "/opt/eos/xrootd/bin/xrdcopy" } if opt.URL == "" { diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go index ab1337a70c..154dcdb1c5 100644 --- a/pkg/eosclient/eosgrpc/eosgrpc.go +++ b/pkg/eosclient/eosgrpc/eosgrpc.go @@ -75,7 +75,7 @@ type Options struct { SingleUsername string // Location of the xrdcopy binary. - // Default is /usr/bin/xrdcopy. + // Default is /opt/eos/xrootd/bin/xrdcopy. XrdcopyBinary string // URL of the EOS MGM. @@ -107,7 +107,7 @@ func (opt *Options) init() { } if opt.XrdcopyBinary == "" { - opt.XrdcopyBinary = "/usr/bin/xrdcopy" + opt.XrdcopyBinary = "/opt/eos/xrootd/bin/xrdcopy" } if opt.URL == "" { diff --git a/pkg/storage/fs/ocis/ocis_test.go b/pkg/storage/fs/ocis/ocis_test.go index f1704ada00..cc13efc343 100644 --- a/pkg/storage/fs/ocis/ocis_test.go +++ b/pkg/storage/fs/ocis/ocis_test.go @@ -23,23 +23,24 @@ package ocis import ( "context" "fmt" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/user" - "github.com/stretchr/testify/assert" "io/ioutil" "os" "path" "sync" "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/user" + "github.com/stretchr/testify/assert" ) // TestLackAdvisoryLocks demonstrates that access to a file // is not mutually exclusive on the oCIS storage. var ( config = make(map[string]interface{}) - ctx context.Context - f, f1 *os.File + ctx context.Context + f, f1 *os.File tmpDir string ) @@ -76,19 +77,19 @@ func TestMain(m *testing.M) { } // do not do this. Prepare f0 - if err = ioutil.WriteFile(fmt.Sprintf("%s/%s", tmpDir,"f.lol"), []byte("test"), 0644); err != nil { + if err = ioutil.WriteFile(fmt.Sprintf("%s/%s", tmpDir, "f.lol"), []byte("test"), 0644); err != nil { panic(err) } - f, err = os.Open(fmt.Sprintf("%s/%s", tmpDir,"f.lol")) + f, err = os.Open(fmt.Sprintf("%s/%s", tmpDir, "f.lol")) if err != nil { panic(err) } // do not do this. Prepare f1 - if err = ioutil.WriteFile(fmt.Sprintf("%s/%s", tmpDir,"f1.lol"), []byte("another run"), 0644); err != nil { + if err = ioutil.WriteFile(fmt.Sprintf("%s/%s", tmpDir, "f1.lol"), []byte("another run"), 0644); err != nil { panic(err) } - f1, err = os.Open(fmt.Sprintf("%s/%s", tmpDir,"f1.lol")) + f1, err = os.Open(fmt.Sprintf("%s/%s", tmpDir, "f1.lol")) if err != nil { panic(err) } @@ -155,8 +156,8 @@ func TestParallelMkcol(t *testing.T) { t.Error(err) } - for i:=0; i< 10; i++ { - t.Run("", func (t *testing.T) { + for i := 0; i < 10; i++ { + t.Run("", func(t *testing.T) { t.Parallel() if err := ofs.CreateDir(ctx, "fightforit"); err != nil { rinfo, err := ofs.GetMD(ctx, &provider.Reference{ diff --git a/pkg/storage/utils/eosfs/config.go b/pkg/storage/utils/eosfs/config.go index 7b867403d6..a05f41c843 100644 --- a/pkg/storage/utils/eosfs/config.go +++ b/pkg/storage/utils/eosfs/config.go @@ -38,7 +38,7 @@ type Config struct { EosBinary string `mapstructure:"eos_binary"` // Location of the xrdcopy binary. - // Default is /usr/bin/xrdcopy. + // Default is /opt/eos/xrootd/bin/xrdcopy. XrdcopyBinary string `mapstructure:"xrdcopy_binary"` // URL of the Master EOS MGM. diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 0aeec73649..b441ab5a04 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -86,7 +86,7 @@ func (c *Config) init() { } if c.XrdcopyBinary == "" { - c.XrdcopyBinary = "/usr/bin/xrdcopy" + c.XrdcopyBinary = "/opt/eos/xrootd/bin/xrdcopy" } if c.MasterURL == "" { From e77a1b6fb651affa8ace352d9cb31113f19b0b26 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 10 Feb 2021 15:53:57 +0100 Subject: [PATCH 2/2] update changelog entry --- changelog/unreleased/xrdcopy-default.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/xrdcopy-default.md b/changelog/unreleased/xrdcopy-default.md index 98ad4da3a9..530e23d5ec 100644 --- a/changelog/unreleased/xrdcopy-default.md +++ b/changelog/unreleased/xrdcopy-default.md @@ -1,3 +1,3 @@ Enhancement: Align default location for xrdcopy binary -https://github.com/cs3org/reva/pull/1431 +https://github.com/cs3org/reva/pull/1467