From d0b9ba720bc99c66dbd39e79f75bffc4218caa9c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 6 Sep 2023 15:43:13 +0200 Subject: [PATCH 01/25] remove unused fs and tracing --- cmd/revad/runtime/grpc.go | 8 - cmd/revad/runtime/runtime.go | 3 +- internal/grpc/interceptors/appctx/appctx.go | 21 +- .../grpc/services/gateway/storageprovider.go | 6 - .../publicstorageprovider.go | 41 - .../storageprovider/storageprovider.go | 10 - internal/http/interceptors/appctx/appctx.go | 11 +- internal/http/services/owncloud/ocdav/copy.go | 9 +- .../http/services/owncloud/ocdav/delete.go | 7 - internal/http/services/owncloud/ocdav/get.go | 9 +- internal/http/services/owncloud/ocdav/head.go | 9 +- .../http/services/owncloud/ocdav/mkcol.go | 9 +- internal/http/services/owncloud/ocdav/move.go | 9 +- .../http/services/owncloud/ocdav/propfind.go | 17 +- .../http/services/owncloud/ocdav/proppatch.go | 9 +- .../services/owncloud/ocdav/publicfile.go | 9 +- internal/http/services/owncloud/ocdav/put.go | 9 +- .../http/services/owncloud/ocdav/trashbin.go | 13 +- internal/http/services/owncloud/ocdav/tus.go | 9 +- .../http/services/owncloud/ocdav/versions.go | 13 +- pkg/rgrpc/todo/pool/pool.go | 20 - pkg/rhttp/rhttp.go | 17 +- pkg/storage/fs/loader/loader.go | 3 - pkg/storage/fs/ocis/blobstore/blobstore.go | 83 -- .../fs/ocis/blobstore/blobstore_suite_test.go | 31 - .../fs/ocis/blobstore/blobstore_test.go | 117 -- pkg/storage/fs/ocis/ocis.go | 50 - pkg/storage/fs/ocis/ocis_suite_test.go | 31 - pkg/storage/fs/ocis/ocis_test.go | 60 -- pkg/storage/fs/s3/s3.go | 702 ------------ pkg/storage/fs/s3/upload.go | 31 - pkg/storage/fs/s3ng/blobstore/blobstore.go | 97 -- pkg/storage/fs/s3ng/option.go | 64 -- pkg/storage/fs/s3ng/option_test.go | 71 -- pkg/storage/fs/s3ng/s3ng.go | 53 - pkg/storage/fs/s3ng/s3ng_suite_test.go | 31 - pkg/storage/fs/s3ng/s3ng_test.go | 70 -- .../utils/decomposedfs/decomposedfs.go | 547 ---------- .../decomposedfs_concurrency_test.go | 139 --- .../decomposedfs/decomposedfs_suite_test.go | 31 - .../utils/decomposedfs/decomposedfs_test.go | 88 -- pkg/storage/utils/decomposedfs/grants.go | 190 ---- pkg/storage/utils/decomposedfs/grants_test.go | 164 --- pkg/storage/utils/decomposedfs/lookup.go | 227 ---- pkg/storage/utils/decomposedfs/lookup_test.go | 118 --- pkg/storage/utils/decomposedfs/metadata.go | 203 ---- .../decomposedfs/mocks/PermissionsChecker.go | 79 -- pkg/storage/utils/decomposedfs/mocks/Tree.go | 291 ----- pkg/storage/utils/decomposedfs/node/node.go | 995 ------------------ .../decomposedfs/node/node_suite_test.go | 31 - .../utils/decomposedfs/node/node_test.go | 195 ---- .../utils/decomposedfs/node/node_unix.go | 34 - .../utils/decomposedfs/node/node_windows.go | 38 - .../utils/decomposedfs/node/permissions.go | 295 ------ .../decomposedfs/node/permissions_darwin.go | 37 - .../decomposedfs/node/permissions_unix.go | 37 - .../utils/decomposedfs/options/options.go | 84 -- .../options/options_suite_test.go | 31 - .../decomposedfs/options/options_test.go | 59 -- pkg/storage/utils/decomposedfs/recycle.go | 344 ------ pkg/storage/utils/decomposedfs/revisions.go | 193 ---- pkg/storage/utils/decomposedfs/spaces.go | 433 -------- .../utils/decomposedfs/testhelpers/helpers.go | 209 ---- .../decomposedfs/tree/mocks/Blobstore.go | 83 -- pkg/storage/utils/decomposedfs/tree/tree.go | 867 --------------- .../decomposedfs/tree/tree_suite_test.go | 31 - .../utils/decomposedfs/tree/tree_test.go | 322 ------ pkg/storage/utils/decomposedfs/upload.go | 727 ------------- pkg/storage/utils/decomposedfs/upload_test.go | 295 ------ .../utils/decomposedfs/xattrs/xattrs.go | 104 -- pkg/trace/trace.go | 110 -- 71 files changed, 31 insertions(+), 9362 deletions(-) delete mode 100644 pkg/storage/fs/ocis/blobstore/blobstore.go delete mode 100644 pkg/storage/fs/ocis/blobstore/blobstore_suite_test.go delete mode 100644 pkg/storage/fs/ocis/blobstore/blobstore_test.go delete mode 100644 pkg/storage/fs/ocis/ocis.go delete mode 100644 pkg/storage/fs/ocis/ocis_suite_test.go delete mode 100644 pkg/storage/fs/ocis/ocis_test.go delete mode 100644 pkg/storage/fs/s3/s3.go delete mode 100644 pkg/storage/fs/s3/upload.go delete mode 100644 pkg/storage/fs/s3ng/blobstore/blobstore.go delete mode 100644 pkg/storage/fs/s3ng/option.go delete mode 100644 pkg/storage/fs/s3ng/option_test.go delete mode 100644 pkg/storage/fs/s3ng/s3ng.go delete mode 100644 pkg/storage/fs/s3ng/s3ng_suite_test.go delete mode 100644 pkg/storage/fs/s3ng/s3ng_test.go delete mode 100644 pkg/storage/utils/decomposedfs/decomposedfs.go delete mode 100644 pkg/storage/utils/decomposedfs/decomposedfs_concurrency_test.go delete mode 100644 pkg/storage/utils/decomposedfs/decomposedfs_suite_test.go delete mode 100644 pkg/storage/utils/decomposedfs/decomposedfs_test.go delete mode 100644 pkg/storage/utils/decomposedfs/grants.go delete mode 100644 pkg/storage/utils/decomposedfs/grants_test.go delete mode 100644 pkg/storage/utils/decomposedfs/lookup.go delete mode 100644 pkg/storage/utils/decomposedfs/lookup_test.go delete mode 100644 pkg/storage/utils/decomposedfs/metadata.go delete mode 100644 pkg/storage/utils/decomposedfs/mocks/PermissionsChecker.go delete mode 100644 pkg/storage/utils/decomposedfs/mocks/Tree.go delete mode 100644 pkg/storage/utils/decomposedfs/node/node.go delete mode 100644 pkg/storage/utils/decomposedfs/node/node_suite_test.go delete mode 100644 pkg/storage/utils/decomposedfs/node/node_test.go delete mode 100644 pkg/storage/utils/decomposedfs/node/node_unix.go delete mode 100644 pkg/storage/utils/decomposedfs/node/node_windows.go delete mode 100644 pkg/storage/utils/decomposedfs/node/permissions.go delete mode 100644 pkg/storage/utils/decomposedfs/node/permissions_darwin.go delete mode 100644 pkg/storage/utils/decomposedfs/node/permissions_unix.go delete mode 100644 pkg/storage/utils/decomposedfs/options/options.go delete mode 100644 pkg/storage/utils/decomposedfs/options/options_suite_test.go delete mode 100644 pkg/storage/utils/decomposedfs/options/options_test.go delete mode 100644 pkg/storage/utils/decomposedfs/recycle.go delete mode 100644 pkg/storage/utils/decomposedfs/revisions.go delete mode 100644 pkg/storage/utils/decomposedfs/spaces.go delete mode 100644 pkg/storage/utils/decomposedfs/testhelpers/helpers.go delete mode 100644 pkg/storage/utils/decomposedfs/tree/mocks/Blobstore.go delete mode 100644 pkg/storage/utils/decomposedfs/tree/tree.go delete mode 100644 pkg/storage/utils/decomposedfs/tree/tree_suite_test.go delete mode 100644 pkg/storage/utils/decomposedfs/tree/tree_test.go delete mode 100644 pkg/storage/utils/decomposedfs/upload.go delete mode 100644 pkg/storage/utils/decomposedfs/upload_test.go delete mode 100644 pkg/storage/utils/decomposedfs/xattrs/xattrs.go delete mode 100644 pkg/trace/trace.go diff --git a/cmd/revad/runtime/grpc.go b/cmd/revad/runtime/grpc.go index 5cd1b4b195..35a70f8a24 100644 --- a/cmd/revad/runtime/grpc.go +++ b/cmd/revad/runtime/grpc.go @@ -28,10 +28,8 @@ import ( "github.com/cs3org/reva/internal/grpc/interceptors/token" "github.com/cs3org/reva/internal/grpc/interceptors/useragent" "github.com/cs3org/reva/pkg/rgrpc" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/pkg/errors" "github.com/rs/zerolog" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" ) @@ -81,12 +79,6 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, logger.Info().Msgf("rgrpc: chaining grpc unary interceptor %s with priority %d", t.Name, t.Priority) } - unaryInterceptors = append(unaryInterceptors, - otelgrpc.UnaryServerInterceptor( - otelgrpc.WithTracerProvider(rtrace.Provider), - otelgrpc.WithPropagators(rtrace.Propagator)), - ) - unaryInterceptors = append([]grpc.UnaryServerInterceptor{ appctx.NewUnary(*logger), token.NewUnary(), diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 40b1df3670..d3627d8b80 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -41,7 +41,6 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rserverless" "github.com/cs3org/reva/pkg/sharedconf" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils/list" "github.com/cs3org/reva/pkg/utils/maps" netutil "github.com/cs3org/reva/pkg/utils/net" @@ -349,7 +348,7 @@ func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { func initTracing(conf *config.Core) { if conf.TracingEnabled { - rtrace.SetTraceProvider(conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName) + //TODO(labkode): init this } } diff --git a/internal/grpc/interceptors/appctx/appctx.go b/internal/grpc/interceptors/appctx/appctx.go index 321ef153a3..66827a295c 100644 --- a/internal/grpc/interceptors/appctx/appctx.go +++ b/internal/grpc/interceptors/appctx/appctx.go @@ -22,23 +22,14 @@ import ( "context" "github.com/cs3org/reva/pkg/appctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" - "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" ) // NewUnary returns a new unary interceptor that creates the application context. func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - span := trace.SpanFromContext(ctx) - defer span.End() - if !span.SpanContext().HasTraceID() { - ctx, span = rtrace.Provider.Tracer("grpc").Start(ctx, "grpc unary") - } - - sub := log.With().Str("traceid", span.SpanContext().TraceID().String()).Logger() - ctx = appctx.WithLogger(ctx, &sub) + ctx = appctx.WithLogger(ctx, &log) res, err := handler(ctx, req) return res, err } @@ -50,15 +41,7 @@ func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { func NewStream(log zerolog.Logger) grpc.StreamServerInterceptor { interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() - span := trace.SpanFromContext(ctx) - defer span.End() - - if !span.SpanContext().HasTraceID() { - ctx, span = rtrace.Provider.Tracer("grpc").Start(ctx, "grpc stream") - } - - sub := log.With().Str("traceid", span.SpanContext().TraceID().String()).Logger() - ctx = appctx.WithLogger(ctx, &sub) + ctx = appctx.WithLogger(ctx, &log) wrapped := newWrappedServerStream(ctx, ss) err := handler(srv, wrapped) diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index a4ec8c4bb2..fb73680beb 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -39,7 +39,6 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/storage/utils/etag" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/golang-jwt/jwt" "github.com/google/uuid" @@ -889,17 +888,12 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide }, nil } - ctx, span := rtrace.Provider.Tracer("reva").Start(ctx, "Delete") - defer span.End() - if !s.inSharedFolder(ctx, p) { return s.delete(ctx, req) } if s.isSharedFolder(ctx, p) { // TODO(labkode): deleting share names should be allowed, means unmounting. - err := errtypes.BadRequest("gateway: cannot delete share folder or share name: path=" + p) - span.RecordError(err) return &provider.DeleteResponse{ Status: status.NewInvalidArg(ctx, "path points to share folder or share name"), }, nil diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 48b7eba838..b6dcacbf7e 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -34,11 +34,9 @@ import ( "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" - "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" @@ -335,14 +333,6 @@ func (s *service) DeleteStorageSpace(ctx context.Context, req *provider.DeleteSt } func (s *service) CreateContainer(ctx context.Context, req *provider.CreateContainerRequest) (*provider.CreateContainerResponse, error) { - ctx, span := rtrace.Provider.Tracer("publicstorageprovider").Start(ctx, "CreateContainer") - defer span.End() - - span.SetAttributes(attribute.KeyValue{ - Key: "reference", - Value: attribute.StringValue(req.Ref.String()), - }) - cs3Ref, _, ls, st, err := s.translatePublicRefToCS3Ref(ctx, req.Ref) switch { case err != nil: @@ -388,14 +378,6 @@ func (s *service) TouchFile(ctx context.Context, req *provider.TouchFileRequest) } func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { - ctx, span := rtrace.Provider.Tracer("publicstorageprovider").Start(ctx, "Delete") - defer span.End() - - span.SetAttributes(attribute.KeyValue{ - Key: "reference", - Value: attribute.StringValue(req.Ref.String()), - }) - cs3Ref, _, ls, st, err := s.translatePublicRefToCS3Ref(ctx, req.Ref) switch { case err != nil: @@ -428,20 +410,6 @@ func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*pro } func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provider.MoveResponse, error) { - ctx, span := rtrace.Provider.Tracer("publicstorageprovider").Start(ctx, "Move") - defer span.End() - - span.SetAttributes( - attribute.KeyValue{ - Key: "source", - Value: attribute.StringValue(req.Source.String()), - }, - attribute.KeyValue{ - Key: "destination", - Value: attribute.StringValue(req.Destination.String()), - }, - ) - cs3RefSource, tknSource, ls, st, err := s.translatePublicRefToCS3Ref(ctx, req.Source) switch { case err != nil: @@ -491,15 +459,6 @@ func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provide } func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) { - ctx, span := rtrace.Provider.Tracer("publicstorageprovider").Start(ctx, "Stat") - defer span.End() - - span.SetAttributes( - attribute.KeyValue{ - Key: "source", - Value: attribute.StringValue(req.Ref.String()), - }) - var ( tkn string relativePath string diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 8a70d81629..257f2b40f4 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -41,12 +41,10 @@ import ( "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/google/uuid" "github.com/pkg/errors" - "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" ) @@ -781,14 +779,6 @@ func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provide } func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) { - ctx, span := rtrace.Provider.Tracer("reva").Start(ctx, "stat") - defer span.End() - - span.SetAttributes(attribute.KeyValue{ - Key: "reference", - Value: attribute.StringValue(req.Ref.String()), - }) - newRef, err := s.unwrap(ctx, req.Ref) if err != nil { // The path might be a virtual view; handle that case diff --git a/internal/http/interceptors/appctx/appctx.go b/internal/http/interceptors/appctx/appctx.go index a5eac9c3c7..0ceb43f9bc 100644 --- a/internal/http/interceptors/appctx/appctx.go +++ b/internal/http/interceptors/appctx/appctx.go @@ -25,9 +25,7 @@ import ( "net/http" "github.com/cs3org/reva/pkg/appctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" - "go.opentelemetry.io/otel/trace" ) // New returns a new HTTP middleware that stores the log @@ -42,14 +40,7 @@ func New(log zerolog.Logger) func(http.Handler) http.Handler { func handler(log zerolog.Logger, h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - span := trace.SpanFromContext(ctx) - defer span.End() - if !span.SpanContext().HasTraceID() { - ctx, span = rtrace.Provider.Tracer("http").Start(ctx, "http interceptor") - } - - sub := log.With().Str("traceid", span.SpanContext().TraceID().String()).Logger() - ctx = appctx.WithLogger(ctx, &sub) + ctx = appctx.WithLogger(ctx, &log) r = r.WithContext(ctx) h.ServeHTTP(w, r) }) diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index 8277fa2370..3dfdd9a699 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/router" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/rs/zerolog" ) @@ -49,9 +48,7 @@ type copy struct { type intermediateDirRefFunc func() (*provider.Reference, *rpc.Status, error) func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "copy") - defer span.End() - + ctx := r.Context() if s.c.EnableHTTPTpc { if r.Header.Get("Source") != "" { // HTTP Third-Party Copy Pull mode @@ -272,9 +269,7 @@ func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClie } func (s *svc) handleSpacesCopy(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "spaces_copy") - defer span.End() - + ctx := r.Context() dst, err := extractDestination(r) if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/internal/http/services/owncloud/ocdav/delete.go b/internal/http/services/owncloud/ocdav/delete.go index 8be2e37443..80bde692d9 100644 --- a/internal/http/services/owncloud/ocdav/delete.go +++ b/internal/http/services/owncloud/ocdav/delete.go @@ -27,7 +27,6 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" ) @@ -47,13 +46,9 @@ func (s *svc) handleDelete(ctx context.Context, w http.ResponseWriter, r *http.R return } - ctx, span := rtrace.Provider.Tracer("reva").Start(ctx, "delete") - defer span.End() - req := &provider.DeleteRequest{Ref: ref} res, err := client.Delete(ctx, req) if err != nil { - span.RecordError(err) log.Error().Err(err).Msg("error performing delete grpc request") w.WriteHeader(http.StatusInternalServerError) return @@ -96,8 +91,6 @@ func (s *svc) handleDelete(ctx context.Context, w http.ResponseWriter, r *http.R func (s *svc) handleSpacesDelete(w http.ResponseWriter, r *http.Request, spaceID string) { ctx := r.Context() - ctx, span := rtrace.Provider.Tracer("reva").Start(ctx, "spaces_delete") - defer span.End() sublog := appctx.GetLogger(ctx).With().Logger() // retrieve a specific storage space diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 338b7bc883..eedb923fbb 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -34,16 +34,13 @@ import ( "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) func (s *svc) handlePathGet(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "get") - defer span.End() - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) sublog := appctx.GetLogger(ctx).With().Str("path", fn).Str("svc", "ocdav").Str("handler", "get").Logger() @@ -160,9 +157,7 @@ func (s *svc) handleGet(ctx context.Context, w http.ResponseWriter, r *http.Requ } func (s *svc) handleSpacesGet(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "spaces_get") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("path", r.URL.Path).Str("spaceid", spaceID).Str("handler", "get").Logger() // retrieve a specific storage space diff --git a/internal/http/services/owncloud/ocdav/head.go b/internal/http/services/owncloud/ocdav/head.go index ff81eacef9..144eeebc0b 100644 --- a/internal/http/services/owncloud/ocdav/head.go +++ b/internal/http/services/owncloud/ocdav/head.go @@ -31,16 +31,13 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/pkg/appctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) func (s *svc) handlePathHead(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "head") - defer span.End() - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger() @@ -90,9 +87,7 @@ func (s *svc) handleHead(ctx context.Context, w http.ResponseWriter, r *http.Req } func (s *svc) handleSpacesHead(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "spaces_head") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger() spaceRef, status, err := s.lookUpStorageSpaceReference(ctx, spaceID, r.URL.Path) diff --git a/internal/http/services/owncloud/ocdav/mkcol.go b/internal/http/services/owncloud/ocdav/mkcol.go index 1092b7f98d..89c55abcb3 100644 --- a/internal/http/services/owncloud/ocdav/mkcol.go +++ b/internal/http/services/owncloud/ocdav/mkcol.go @@ -27,14 +27,11 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" ) func (s *svc) handlePathMkcol(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "mkcol") - defer span.End() - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) for _, r := range nameRules { if !r.Test(fn) { @@ -51,9 +48,7 @@ func (s *svc) handlePathMkcol(w http.ResponseWriter, r *http.Request, ns string) } func (s *svc) handleSpacesMkCol(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), "spaces_mkcol") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("path", r.URL.Path).Str("spaceid", spaceID).Str("handler", "mkcol").Logger() parentRef, rpcStatus, err := s.lookUpStorageSpaceReference(ctx, spaceID, path.Dir(r.URL.Path)) diff --git a/internal/http/services/owncloud/ocdav/move.go b/internal/http/services/owncloud/ocdav/move.go index c159c2a733..3c82ab36c4 100644 --- a/internal/http/services/owncloud/ocdav/move.go +++ b/internal/http/services/owncloud/ocdav/move.go @@ -29,15 +29,12 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/router" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) func (s *svc) handlePathMove(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "move") - defer span.End() - + ctx := r.Context() srcPath := path.Join(ns, r.URL.Path) dstPath, err := extractDestination(r) if err != nil { @@ -67,9 +64,7 @@ func (s *svc) handlePathMove(w http.ResponseWriter, r *http.Request, ns string) } func (s *svc) handleSpacesMove(w http.ResponseWriter, r *http.Request, srcSpaceID string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "spaces_move") - defer span.End() - + ctx := r.Context() dst, err := extractDestination(r) if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index fda9e06e17..51c5724358 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -45,12 +45,9 @@ import ( "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/share" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" ) const ( @@ -70,11 +67,7 @@ const ( // ns is the namespace that is prefixed to the path in the cs3 namespace. func (s *svc) handlePathPropfind(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("reva").Start(r.Context(), fmt.Sprintf("%s %v", r.Method, r.URL.Path)) - defer span.End() - - span.SetAttributes(attribute.String("component", "ocdav")) - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger() @@ -97,8 +90,7 @@ func (s *svc) handlePathPropfind(w http.ResponseWriter, r *http.Request, ns stri } func (s *svc) handleSpacesPropfind(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "spaces_propfind") - defer span.End() + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("path", r.URL.Path).Str("spaceid", spaceID).Logger() @@ -144,9 +136,6 @@ func (s *svc) handleSpacesPropfind(w http.ResponseWriter, r *http.Request, space } func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, namespace string, pf propfindXML, parentInfo *provider.ResourceInfo, resourceInfos []*provider.ResourceInfo, log zerolog.Logger) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(ctx, "propfind_response") - defer span.End() - linkFilters := make([]*link.ListPublicSharesRequest_Filter, 0, len(resourceInfos)) shareFilters := make([]*collaboration.Filter, 0, len(resourceInfos)) for i := range resourceInfos { @@ -170,7 +159,6 @@ func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *ht } } else { log.Error().Err(err).Msg("propfindResponse: couldn't list public shares") - span.SetStatus(codes.Error, err.Error()) } var usershares map[string]struct{} @@ -182,7 +170,6 @@ func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *ht } } else { log.Error().Err(err).Msg("propfindResponse: couldn't list user shares") - span.SetStatus(codes.Error, err.Error()) } propRes, err := s.multistatusResponse(ctx, &pf, resourceInfos, namespace, usershares, linkshares) diff --git a/internal/http/services/owncloud/ocdav/proppatch.go b/internal/http/services/owncloud/ocdav/proppatch.go index cd246f821e..cfb0b1b513 100644 --- a/internal/http/services/owncloud/ocdav/proppatch.go +++ b/internal/http/services/owncloud/ocdav/proppatch.go @@ -31,15 +31,12 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/pkg/errors" "github.com/rs/zerolog" ) func (s *svc) handlePathProppatch(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "proppatch") - defer span.End() - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger() @@ -104,9 +101,7 @@ func (s *svc) handlePathProppatch(w http.ResponseWriter, r *http.Request, ns str } func (s *svc) handleSpacesProppatch(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "spaces_proppatch") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("path", r.URL.Path).Str("spaceid", spaceID).Logger() pp, status, err := readProppatch(r.Body) diff --git a/internal/http/services/owncloud/ocdav/publicfile.go b/internal/http/services/owncloud/ocdav/publicfile.go index e25a82f1e3..b9b5fbb169 100644 --- a/internal/http/services/owncloud/ocdav/publicfile.go +++ b/internal/http/services/owncloud/ocdav/publicfile.go @@ -27,7 +27,6 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/router" - rtrace "github.com/cs3org/reva/pkg/trace" ) // PublicFileHandler handles requests on a shared file. it needs to be wrapped in a collection. @@ -86,9 +85,7 @@ func (h *PublicFileHandler) Handler(s *svc) http.Handler { } func (s *svc) adjustResourcePathInURL(w http.ResponseWriter, r *http.Request) bool { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "adjustResourcePathInURL") - defer span.End() - + ctx := r.Context() // find actual file name tokenStatInfo := ctx.Value(tokenStatInfoKey{}).(*provider.ResourceInfo) sublog := appctx.GetLogger(ctx).With().Interface("tokenStatInfo", tokenStatInfo).Logger() @@ -125,9 +122,7 @@ func (s *svc) adjustResourcePathInURL(w http.ResponseWriter, r *http.Request) bo // ns is the namespace that is prefixed to the path in the cs3 namespace. func (s *svc) handlePropfindOnToken(w http.ResponseWriter, r *http.Request, ns string, onContainer bool) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "token_propfind") - defer span.End() - + ctx := r.Context() tokenStatInfo := ctx.Value(tokenStatInfoKey{}).(*provider.ResourceInfo) sublog := appctx.GetLogger(ctx).With().Interface("tokenStatInfo", tokenStatInfo).Logger() sublog.Debug().Msg("handlePropfindOnToken") diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index c2ee4fffc0..9f18985259 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -38,7 +38,6 @@ import ( "github.com/cs3org/reva/pkg/notification/trigger" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage/utils/chunking" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" @@ -110,9 +109,7 @@ func isContentRange(r *http.Request) bool { } func (s *svc) handlePathPut(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "put") - defer span.End() - + ctx := r.Context() fn := path.Join(ns, r.URL.Path) sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger() @@ -383,9 +380,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ } func (s *svc) handleSpacesPut(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "spaces_put") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger() spaceRef, status, err := s.lookUpStorageSpaceReference(ctx, spaceID, r.URL.Path) diff --git a/internal/http/services/owncloud/ocdav/trashbin.go b/internal/http/services/owncloud/ocdav/trashbin.go index 315d613a32..1bc2b5248b 100644 --- a/internal/http/services/owncloud/ocdav/trashbin.go +++ b/internal/http/services/owncloud/ocdav/trashbin.go @@ -36,7 +36,6 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" ) @@ -163,9 +162,7 @@ func (h *TrashbinHandler) Handler(s *svc) http.Handler { } func (h *TrashbinHandler) listTrashbin(w http.ResponseWriter, r *http.Request, s *svc, u *userpb.User, basePath, key, itemPath string) { - ctx, span := rtrace.Provider.Tracer("trash-bin").Start(r.Context(), "list_trashbin") - defer span.End() - + ctx := r.Context() depth := r.Header.Get(HeaderDepth) if depth == "" { depth = "1" @@ -439,9 +436,7 @@ func (h *TrashbinHandler) itemToPropResponse(ctx context.Context, s *svc, u *use } func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc, u *userpb.User, basePath, dst, key, itemPath string) { - ctx, span := rtrace.Provider.Tracer("trash-bin").Start(r.Context(), "restore") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Logger() overwrite := r.Header.Get(HeaderOverwrite) @@ -587,9 +582,7 @@ func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc // delete has only a key. func (h *TrashbinHandler) delete(w http.ResponseWriter, r *http.Request, s *svc, u *userpb.User, basePath, key, itemPath string) { - ctx, span := rtrace.Provider.Tracer("trash-bin").Start(r.Context(), "erase") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Str("key", key).Logger() client, err := s.getClient() diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index bdc65cdff7..b3a8fa8402 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" @@ -42,9 +41,7 @@ import ( ) func (s *svc) handlePathTusPost(w http.ResponseWriter, r *http.Request, ns string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "tus-post") - defer span.End() - + ctx := r.Context() // read filename from metadata meta := tusd.ParseMetadataHeader(r.Header.Get(HeaderUploadMetadata)) for _, r := range nameRules { @@ -65,9 +62,7 @@ func (s *svc) handlePathTusPost(w http.ResponseWriter, r *http.Request, ns strin } func (s *svc) handleSpacesTusPost(w http.ResponseWriter, r *http.Request, spaceID string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "spaces-tus-post") - defer span.End() - + ctx := r.Context() // read filename from metadata meta := tusd.ParseMetadataHeader(r.Header.Get(HeaderUploadMetadata)) if meta["filename"] == "" { diff --git a/internal/http/services/owncloud/ocdav/versions.go b/internal/http/services/owncloud/ocdav/versions.go index 7c42d2900f..463b4dc8be 100644 --- a/internal/http/services/owncloud/ocdav/versions.go +++ b/internal/http/services/owncloud/ocdav/versions.go @@ -32,7 +32,6 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/storage/utils/downloader" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils/resourceid" ) @@ -88,9 +87,7 @@ func (h *VersionsHandler) Handler(s *svc, rid *provider.ResourceId) http.Handler } func (h *VersionsHandler) doListVersions(w http.ResponseWriter, r *http.Request, s *svc, rid *provider.ResourceId) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "listVersions") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Interface("resourceid", rid).Logger() pf, status, err := readPropfind(r.Body) @@ -189,9 +186,7 @@ func (h *VersionsHandler) doListVersions(w http.ResponseWriter, r *http.Request, } func (h *VersionsHandler) doRestore(w http.ResponseWriter, r *http.Request, s *svc, rid *provider.ResourceId, key string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "restore") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Interface("resourceid", rid).Str("key", key).Logger() client, err := s.getClient() @@ -220,9 +215,7 @@ func (h *VersionsHandler) doRestore(w http.ResponseWriter, r *http.Request, s *s } func (h *VersionsHandler) doDownload(w http.ResponseWriter, r *http.Request, s *svc, rid *provider.ResourceId, key string) { - ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "restore") - defer span.End() - + ctx := r.Context() sublog := appctx.GetLogger(ctx).With().Interface("resourceid", rid).Str("key", key).Logger() client, err := s.getClient() diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 38092d44b6..67904d0328 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -40,8 +40,6 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storageregistry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" - rtrace "github.com/cs3org/reva/pkg/trace" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -92,24 +90,6 @@ func NewConn(options Options) (*grpc.ClientConn, error) { grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(options.MaxCallRecvMsgSize), ), - grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor( - otelgrpc.WithTracerProvider( - rtrace.Provider, - ), - otelgrpc.WithPropagators( - rtrace.Propagator, - ), - )), - grpc.WithUnaryInterceptor( - otelgrpc.UnaryClientInterceptor( - otelgrpc.WithTracerProvider( - rtrace.Provider, - ), - otelgrpc.WithPropagators( - rtrace.Propagator, - ), - ), - ), ) if err != nil { return nil, err diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index 37fd52dd51..c4aafe9c07 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -30,10 +30,8 @@ import ( "github.com/cs3org/reva/cmd/revad/pkg/config" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" - rtrace "github.com/cs3org/reva/pkg/trace" "github.com/pkg/errors" "github.com/rs/zerolog" - "go.opentelemetry.io/otel/propagation" ) type Config func(*Server) @@ -181,8 +179,7 @@ func (s *Server) GracefulStop() error { func (s *Server) registerServices() { for name, svc := range s.Services { // instrument services with opencensus tracing. - h := traceHandler(name, svc.Handler()) - s.handlers[svc.Prefix()] = h + s.handlers[svc.Prefix()] = svc.Handler() s.svcs[svc.Prefix()] = svc s.unprotected = append(s.unprotected, getUnprotected(svc.Prefix(), svc.Unprotected())...) s.log.Info().Msgf("http service enabled: %s@/%s", name, svc.Prefix()) @@ -282,15 +279,3 @@ func (s *Server) getHandler() (http.Handler, error) { return handler, nil } - -func traceHandler(name string, h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := rtrace.Propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) - t := rtrace.Provider.Tracer("reva") - ctx, span := t.Start(ctx, name) - defer span.End() - - rtrace.Propagator.Inject(ctx, propagation.HeaderCarrier(r.Header)) - h.ServeHTTP(w, r.WithContext(ctx)) - }) -} diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 0f4752ac85..e9dfad143f 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -31,10 +31,7 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/local" _ "github.com/cs3org/reva/pkg/storage/fs/localhome" _ "github.com/cs3org/reva/pkg/storage/fs/nextcloud" - _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/owncloudsql" - _ "github.com/cs3org/reva/pkg/storage/fs/s3" - _ "github.com/cs3org/reva/pkg/storage/fs/s3ng" // Add your own here. ) diff --git a/pkg/storage/fs/ocis/blobstore/blobstore.go b/pkg/storage/fs/ocis/blobstore/blobstore.go deleted file mode 100644 index aa2bb77d24..0000000000 --- a/pkg/storage/fs/ocis/blobstore/blobstore.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore - -import ( - "bufio" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -// Blobstore provides an interface to an filesystem based blobstore. -type Blobstore struct { - root string -} - -// New returns a new Blobstore. -func New(root string) (*Blobstore, error) { - err := os.MkdirAll(root, 0700) - if err != nil { - return nil, err - } - - return &Blobstore{ - root: root, - }, nil -} - -// Upload stores some data in the blobstore under the given key. -func (bs *Blobstore) Upload(key string, data io.Reader) error { - f, err := os.OpenFile(bs.path(key), os.O_CREATE|os.O_WRONLY, 0700) - if err != nil { - return errors.Wrapf(err, "could not open blob '%s' for writing", key) - } - - w := bufio.NewWriter(f) - _, err = w.ReadFrom(data) - if err != nil { - return errors.Wrapf(err, "could not write blob '%s'", key) - } - - return w.Flush() -} - -// Download retrieves a blob from the blobstore for reading. -func (bs *Blobstore) Download(key string) (io.ReadCloser, error) { - file, err := os.Open(bs.path(key)) - if err != nil { - return nil, errors.Wrapf(err, "could not read blob '%s'", key) - } - return file, nil -} - -// Delete deletes a blob from the blobstore. -func (bs *Blobstore) Delete(key string) error { - err := os.Remove(bs.path(key)) - if err != nil { - return errors.Wrapf(err, "could not delete blob '%s'", key) - } - return nil -} - -func (bs *Blobstore) path(key string) string { - return filepath.Join(bs.root, filepath.Clean(filepath.Join("/", key))) -} diff --git a/pkg/storage/fs/ocis/blobstore/blobstore_suite_test.go b/pkg/storage/fs/ocis/blobstore/blobstore_suite_test.go deleted file mode 100644 index ae3e86e9f0..0000000000 --- a/pkg/storage/fs/ocis/blobstore/blobstore_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestBlobstore(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Blobstore Suite") -} diff --git a/pkg/storage/fs/ocis/blobstore/blobstore_test.go b/pkg/storage/fs/ocis/blobstore/blobstore_test.go deleted file mode 100644 index 17e7e77419..0000000000 --- a/pkg/storage/fs/ocis/blobstore/blobstore_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore_test - -import ( - "bytes" - "io" - "os" - "path" - - "github.com/cs3org/reva/pkg/storage/fs/ocis/blobstore" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Blobstore", func() { - var ( - tmpRoot string - key string - blobPath string - data []byte - - bs *blobstore.Blobstore - ) - - BeforeEach(func() { - var err error - tmpRoot, err = helpers.TempDir("reva-unit-tests-*-root") - Expect(err).ToNot(HaveOccurred()) - - data = []byte("1234567890") - key = "foo" - blobPath = path.Join(tmpRoot, "blobs", key) - - bs, err = blobstore.New(path.Join(tmpRoot, "blobs")) - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - if tmpRoot != "" { - os.RemoveAll(tmpRoot) - } - }) - - It("creates the root directory if it doesn't exist", func() { - _, err := os.Stat(path.Join(tmpRoot, "blobs")) - Expect(err).ToNot(HaveOccurred()) - }) - - Describe("Upload", func() { - It("writes the blob", func() { - err := bs.Upload(key, bytes.NewReader(data)) - Expect(err).ToNot(HaveOccurred()) - - writtenBytes, err := os.ReadFile(blobPath) - Expect(err).ToNot(HaveOccurred()) - Expect(writtenBytes).To(Equal(data)) - }) - }) - - Context("with an existing blob", func() { - BeforeEach(func() { - Expect(os.WriteFile(blobPath, data, 0700)).To(Succeed()) - }) - - Describe("Download", func() { - It("cleans the key", func() { - reader, err := bs.Download("../" + key) - Expect(err).ToNot(HaveOccurred()) - - readData, err := io.ReadAll(reader) - Expect(err).ToNot(HaveOccurred()) - Expect(readData).To(Equal(data)) - }) - - It("returns a reader to the blob", func() { - reader, err := bs.Download(key) - Expect(err).ToNot(HaveOccurred()) - - readData, err := io.ReadAll(reader) - Expect(err).ToNot(HaveOccurred()) - Expect(readData).To(Equal(data)) - }) - }) - - Describe("Delete", func() { - It("deletes the blob", func() { - _, err := os.Stat(blobPath) - Expect(err).ToNot(HaveOccurred()) - - err = bs.Delete(key) - Expect(err).ToNot(HaveOccurred()) - - _, err = os.Stat(blobPath) - Expect(err).To(HaveOccurred()) - }) - }) - }) - -}) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go deleted file mode 100644 index 252c351bc2..0000000000 --- a/pkg/storage/fs/ocis/ocis.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package ocis - -import ( - "context" - "path" - - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/fs/ocis/blobstore" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" -) - -func init() { - registry.Register("ocis", New) -} - -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - o, err := options.New(m) - if err != nil { - return nil, err - } - - bs, err := blobstore.New(path.Join(o.Root, "blobs")) - if err != nil { - return nil, err - } - - return decomposedfs.NewDefault(m, bs) -} diff --git a/pkg/storage/fs/ocis/ocis_suite_test.go b/pkg/storage/fs/ocis/ocis_suite_test.go deleted file mode 100644 index 21cb044f2d..0000000000 --- a/pkg/storage/fs/ocis/ocis_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package ocis_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestOcis(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Ocis Suite") -} diff --git a/pkg/storage/fs/ocis/ocis_test.go b/pkg/storage/fs/ocis/ocis_test.go deleted file mode 100644 index be7e255882..0000000000 --- a/pkg/storage/fs/ocis/ocis_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package ocis_test - -import ( - "context" - "os" - - "github.com/cs3org/reva/pkg/storage/fs/ocis" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Ocis", func() { - var ( - options map[string]interface{} - tmpRoot string - ) - - BeforeEach(func() { - tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") - Expect(err).ToNot(HaveOccurred()) - - options = map[string]interface{}{ - "root": tmpRoot, - "enable_home": true, - "share_folder": "/Shares", - } - }) - - AfterEach(func() { - if tmpRoot != "" { - os.RemoveAll(tmpRoot) - } - }) - - Describe("New", func() { - It("returns a new instance", func() { - _, err := ocis.New(context.Background(), options) - Expect(err).ToNot(HaveOccurred()) - }) - }) -}) diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go deleted file mode 100644 index 3b4983ee3b..0000000000 --- a/pkg/storage/fs/s3/s3.go +++ /dev/null @@ -1,702 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3 - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - "path" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/mime" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/utils/cfg" - "github.com/pkg/errors" -) - -func init() { - registry.Register("s3", New) -} - -type config struct { - Region string `mapstructure:"region"` - AccessKey string `mapstructure:"access_key"` - SecretKey string `mapstructure:"secret_key"` - Endpoint string `mapstructure:"endpoint"` - Bucket string `mapstructure:"bucket"` - Prefix string `mapstructure:"prefix"` -} - -// New returns an implementation to of the storage.FS interface that talk to -// a s3 api. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - var c config - if err := cfg.Decode(m, &c); err != nil { - return nil, err - } - - awsConfig := aws.NewConfig(). - WithHTTPClient(http.DefaultClient). - WithMaxRetries(aws.UseServiceDefaultRetries). - WithLogger(aws.NewDefaultLogger()). - WithLogLevel(aws.LogOff). - WithSleepDelay(time.Sleep). - WithCredentials(credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")). - WithEndpoint(c.Endpoint). - WithS3ForcePathStyle(true). - WithDisableSSL(true) - - if c.Region != "" { - awsConfig.WithRegion(c.Region) - } else { - awsConfig.WithRegion("us-east-1") - } - - sess, err := session.NewSession(awsConfig) - if err != nil { - return nil, err - } - if sess == nil { - return nil, errors.New("creating the S3 session") - } - - s3Client := s3.New(sess) - - return &s3FS{client: s3Client, config: &c}, nil -} - -func (fs *s3FS) Shutdown(ctx context.Context) error { - return nil -} - -func (fs *s3FS) addRoot(p string) string { - np := path.Join(fs.config.Prefix, p) - return np -} - -func (fs *s3FS) resolve(ctx context.Context, ref *provider.Reference) (string, error) { - if strings.HasPrefix(ref.Path, "/") { - return fs.addRoot(ref.GetPath()), nil - } - - if ref.ResourceId != nil && ref.ResourceId.OpaqueId != "" { - fn := path.Join("/", strings.TrimPrefix(ref.ResourceId.OpaqueId, "fileid-")) - fn = fs.addRoot(fn) - return fn, nil - } - - // reference is invalid - return "", fmt.Errorf("invalid reference %+v", ref) -} - -func (fs *s3FS) removeRoot(np string) string { - p := strings.TrimPrefix(np, fs.config.Prefix) - if p == "" { - p = "/" - } - return p -} - -type s3FS struct { - client *s3.S3 - config *config -} - -// permissionSet returns the permission set for the current user. -func (fs *s3FS) permissionSet(ctx context.Context) *provider.ResourcePermissions { - // TODO fix permissions for share recipients by traversing reading acls up to the root? cache acls for the parent node and reuse it - return &provider.ResourcePermissions{ - // owner has all permissions - AddGrant: true, - CreateContainer: true, - Delete: true, - GetPath: true, - GetQuota: true, - InitiateFileDownload: true, - InitiateFileUpload: true, - ListContainer: true, - ListFileVersions: true, - ListGrants: true, - ListRecycle: true, - Move: true, - PurgeRecycle: true, - RemoveGrant: true, - RestoreFileVersion: true, - RestoreRecycleItem: true, - Stat: true, - UpdateGrant: true, - } -} - -func (fs *s3FS) normalizeObject(ctx context.Context, o *s3.Object, fn string) *provider.ResourceInfo { - fn = fs.removeRoot(path.Join("/", fn)) - isDir := strings.HasSuffix(*o.Key, "/") - md := &provider.ResourceInfo{ - Id: &provider.ResourceId{ - OpaqueId: "fileid-" + strings.TrimPrefix(fn, "/"), - }, - Path: fn, - Type: getResourceType(isDir), - Etag: *o.ETag, - MimeType: mime.Detect(isDir, fn), - PermissionSet: fs.permissionSet(ctx), - Size: uint64(*o.Size), - Mtime: &types.Timestamp{ - Seconds: uint64(o.LastModified.Unix()), - }, - } - appctx.GetLogger(ctx).Debug(). - Interface("object", o). - Interface("metadata", md). - Msg("normalized Object") - return md -} - -func getResourceType(isDir bool) provider.ResourceType { - if isDir { - return provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - return provider.ResourceType_RESOURCE_TYPE_CONTAINER -} - -func (fs *s3FS) normalizeHead(ctx context.Context, o *s3.HeadObjectOutput, fn string) *provider.ResourceInfo { - fn = fs.removeRoot(path.Join("/", fn)) - isDir := strings.HasSuffix(fn, "/") - md := &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: "fileid-" + strings.TrimPrefix(fn, "/")}, - Path: fn, - Type: getResourceType(isDir), - Etag: *o.ETag, - MimeType: mime.Detect(isDir, fn), - PermissionSet: fs.permissionSet(ctx), - Size: uint64(*o.ContentLength), - Mtime: &types.Timestamp{ - Seconds: uint64(o.LastModified.Unix()), - }, - } - appctx.GetLogger(ctx).Debug(). - Interface("head", o). - Interface("metadata", md). - Msg("normalized Head") - return md -} -func (fs *s3FS) normalizeCommonPrefix(ctx context.Context, p *s3.CommonPrefix) *provider.ResourceInfo { - fn := fs.removeRoot(path.Join("/", *p.Prefix)) - md := &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: "fileid-" + strings.TrimPrefix(fn, "/")}, - Path: fn, - Type: getResourceType(true), - Etag: "TODO(labkode)", - MimeType: mime.Detect(true, fn), - PermissionSet: fs.permissionSet(ctx), - Size: 0, - Mtime: &types.Timestamp{ - Seconds: 0, - }, - } - appctx.GetLogger(ctx).Debug(). - Interface("prefix", p). - Interface("metadata", md). - Msg("normalized CommonPrefix") - return md -} - -// GetPathByID returns the path pointed by the file id -// In this implementation the file id is that path of the file without the first slash -// thus the file id always points to the filename. -func (fs *s3FS) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { - return path.Join("/", strings.TrimPrefix(id.OpaqueId, "fileid-")), nil -} - -func (fs *s3FS) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { - return nil, errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { - return 0, 0, nil -} - -func (fs *s3FS) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error { - return errtypes.NotSupported("s3: operation not supported") -} - -// GetLock returns an existing lock on the given reference. -func (fs *s3FS) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { - return nil, errtypes.NotSupported("unimplemented") -} - -// SetLock puts a lock on the given reference. -func (fs *s3FS) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - return errtypes.NotSupported("unimplemented") -} - -// RefreshLock refreshes an existing lock on the given reference. -func (fs *s3FS) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock, existingLockID string) error { - return errtypes.NotSupported("unimplemented") -} - -// Unlock removes an existing lock from the given reference. -func (fs *s3FS) Unlock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - return errtypes.NotSupported("unimplemented") -} - -func (fs *s3FS) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { - // TODO(jfd):implement - return errtypes.NotSupported("s3: operation not supported") -} - -func (fs *s3FS) GetHome(ctx context.Context) (string, error) { - return "", errtypes.NotSupported("eos: not supported") -} - -func (fs *s3FS) CreateHome(ctx context.Context) error { - return errtypes.NotSupported("s3fs: not supported") -} - -func (fs *s3FS) CreateDir(ctx context.Context, ref *provider.Reference) error { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, ref) - if err != nil { - return nil - } - - fn = fs.addRoot(fn) + "/" // append / to indicate folder // TODO only if fn does not end in / - - input := &s3.PutObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - ContentType: aws.String("application/octet-stream"), - ContentLength: aws.Int64(0), - } - - result, err := fs.client.PutObject(input) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == s3.ErrCodeNoSuchBucket { - return errtypes.NotFound(ref.Path) - } - } - // FIXME we also need already exists error, webdav expects 405 MethodNotAllowed - return errors.Wrap(err, "s3fs: error creating dir "+ref.Path) - } - - log.Debug().Interface("result", result) // todo cache etag? - return nil -} - -// TouchFile as defined in the storage.FS interface. -func (fs *s3FS) TouchFile(ctx context.Context, ref *provider.Reference) error { - return fmt.Errorf("unimplemented: TouchFile") -} - -func (fs *s3FS) Delete(ctx context.Context, ref *provider.Reference) error { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - // first we need to find out if fn is a dir or a file - - _, err = fs.client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - }) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return errtypes.NotFound(fn) - } - } - // it might be a directory, so we can batch delete the prefix + / - iter := s3manager.NewDeleteListIterator(fs.client, &s3.ListObjectsInput{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(fn + "/"), - }) - batcher := s3manager.NewBatchDeleteWithClient(fs.client) - if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil { - return err - } - // ok, we are done - return nil - } - - // we found an object, let's get rid of it - result, err := fs.client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - }) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return errtypes.NotFound(fn) - } - } - return errors.Wrap(err, "s3fs: error deleting "+fn) - } - - log.Debug().Interface("result", result) - return nil -} - -// CreateStorageSpace creates a storage space. -func (fs *s3FS) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { - return nil, fmt.Errorf("unimplemented: CreateStorageSpace") -} - -func (fs *s3FS) moveObject(ctx context.Context, oldKey string, newKey string) error { - // Copy - // TODO double check CopyObject can deal with >5GB files. - // Docs say we need to use multipart upload: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html - _, err := fs.client.CopyObject(&s3.CopyObjectInput{ - Bucket: aws.String(fs.config.Bucket), - CopySource: aws.String("/" + fs.config.Bucket + oldKey), - Key: aws.String(newKey), - }) - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == s3.ErrCodeNoSuchBucket { - return errtypes.NotFound(oldKey) - } - return err - } - // TODO cache etag and mtime? - - // Delete - _, err = fs.client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(oldKey), - }) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return errtypes.NotFound(oldKey) - } - return err - } - return nil -} - -func (fs *s3FS) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, oldRef) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - newName, err := fs.resolve(ctx, newRef) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - // first we need to find out if fn is a dir or a file - - _, err = fs.client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - }) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return errtypes.NotFound(fn) - } - } - - // move directory - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(fn + "/"), - } - isTruncated := true - - for isTruncated { - output, err := fs.client.ListObjectsV2(input) - if err != nil { - return errors.Wrap(err, "s3FS: error listing "+fn) - } - - for _, o := range output.Contents { - log.Debug(). - Interface("object", *o). - Str("fn", fn). - Msg("found Object") - - err := fs.moveObject(ctx, *o.Key, strings.Replace(*o.Key, fn+"/", newName+"/", 1)) - if err != nil { - return err - } - } - - input.ContinuationToken = output.NextContinuationToken - isTruncated = *output.IsTruncated - } - // ok, we are done - return nil - } - - // move single object - err = fs.moveObject(ctx, fn, newName) - if err != nil { - return err - } - return nil -} - -func (fs *s3FS) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "error resolving ref") - } - - // first try a head, works for files - log.Debug(). - Str("fn", fn). - Msg("trying HEAD") - - input := &s3.HeadObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - } - output, err := fs.client.HeadObject(input) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return nil, errtypes.NotFound(fn) - } - } - log.Debug(). - Str("fn", fn). - Msg("trying to list prefix") - // try by listing parent to find directory - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(fn), - Delimiter: aws.String("/"), // limit to a single directory - } - isTruncated := true - - for isTruncated { - output, err := fs.client.ListObjectsV2(input) - if err != nil { - return nil, errors.Wrap(err, "s3FS: error listing "+fn) - } - - for i := range output.CommonPrefixes { - log.Debug(). - Interface("object", output.CommonPrefixes[i]). - Str("fn", fn). - Msg("found CommonPrefix") - if *output.CommonPrefixes[i].Prefix == fn+"/" { - return fs.normalizeCommonPrefix(ctx, output.CommonPrefixes[i]), nil - } - } - - input.ContinuationToken = output.NextContinuationToken - isTruncated = *output.IsTruncated - } - return nil, errtypes.NotFound(fn) - } - - return fs.normalizeHead(ctx, output, fn), nil -} - -func (fs *s3FS) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { - fn, err := fs.resolve(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "error resolving ref") - } - - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(fs.config.Bucket), - Prefix: aws.String(fn + "/"), - Delimiter: aws.String("/"), // limit to a single directory - } - isTruncated := true - - finfos := []*provider.ResourceInfo{} - - for isTruncated { - output, err := fs.client.ListObjectsV2(input) - if err != nil { - return nil, errors.Wrap(err, "s3FS: error listing "+fn) - } - - for i := range output.CommonPrefixes { - finfos = append(finfos, fs.normalizeCommonPrefix(ctx, output.CommonPrefixes[i])) - } - - for i := range output.Contents { - finfos = append(finfos, fs.normalizeObject(ctx, output.Contents[i], *output.Contents[i].Key)) - } - - input.ContinuationToken = output.NextContinuationToken - isTruncated = *output.IsTruncated - } - // TODO sort fileinfos? - return finfos, nil -} - -func (fs *s3FS) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - upParams := &s3manager.UploadInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - Body: r, - } - uploader := s3manager.NewUploaderWithClient(fs.client) - result, err := uploader.Upload(upParams) - - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == s3.ErrCodeNoSuchBucket { - return errtypes.NotFound(fn) - } - } - return errors.Wrap(err, "s3fs: error creating object "+fn) - } - - log.Debug().Interface("result", result) // todo cache etag? - return nil -} - -func (fs *s3FS) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - log := appctx.GetLogger(ctx) - - fn, err := fs.resolve(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "error resolving ref") - } - - // use GetObject instead of s3manager.Downloader: - // the result.Body is a ReadCloser, which allows streaming - // TODO double check we are not caching bytes in memory - r, err := fs.client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(fn), - }) - if err != nil { - log.Error().Err(err) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case s3.ErrCodeNoSuchBucket: - case s3.ErrCodeNoSuchKey: - return nil, errtypes.NotFound(fn) - } - } - return nil, errors.Wrap(err, "s3fs: error deleting "+fn) - } - return r.Body, nil -} - -func (fs *s3FS) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { - return nil, errtypes.NotSupported("list revisions") -} - -func (fs *s3FS) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - return nil, errtypes.NotSupported("download revision") -} - -func (fs *s3FS) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { - return errtypes.NotSupported("restore revision") -} - -func (fs *s3FS) PurgeRecycleItem(ctx context.Context, kbasePath, key, relativePath string) error { - return errtypes.NotSupported("purge recycle item") -} - -func (fs *s3FS) EmptyRecycle(ctx context.Context) error { - return errtypes.NotSupported("empty recycle") -} - -func (fs *s3FS) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) { - return nil, errtypes.NotSupported("list recycle") -} - -func (fs *s3FS) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { - return errtypes.NotSupported("restore recycle") -} - -func (fs *s3FS) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { - return nil, errtypes.NotSupported("list storage spaces") -} - -// UpdateStorageSpace updates a storage space. -func (fs *s3FS) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { - return nil, errtypes.NotSupported("update storage space") -} diff --git a/pkg/storage/fs/s3/upload.go b/pkg/storage/fs/s3/upload.go deleted file mode 100644 index e3343b38f9..0000000000 --- a/pkg/storage/fs/s3/upload.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3 - -import ( - "context" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/errtypes" -) - -// InitiateUpload returns upload ids corresponding to different protocols it supports. -func (fs *s3FS) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - return nil, errtypes.NotSupported("op not supported") -} diff --git a/pkg/storage/fs/s3ng/blobstore/blobstore.go b/pkg/storage/fs/s3ng/blobstore/blobstore.go deleted file mode 100644 index b35bd67962..0000000000 --- a/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore - -import ( - "context" - "io" - "net/url" - "os" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/pkg/errors" -) - -// Blobstore provides an interface to an s3 compatible blobstore. -type Blobstore struct { - client *minio.Client - - bucket string -} - -// New returns a new Blobstore. -func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, error) { - u, err := url.Parse(endpoint) - if err != nil { - return nil, errors.Wrap(err, "failed to parse s3 endpoint") - } - - useSSL := u.Scheme != "http" - client, err := minio.New(u.Host, &minio.Options{ - Region: region, - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: useSSL, - }) - if err != nil { - return nil, errors.Wrap(err, "failed to setup s3 client") - } - - return &Blobstore{ - client: client, - bucket: bucket, - }, nil -} - -// Upload stores some data in the blobstore under the given key. -func (bs *Blobstore) Upload(key string, reader io.Reader) error { - size := int64(-1) - if file, ok := reader.(*os.File); ok { - info, err := file.Stat() - if err != nil { - return errors.Wrapf(err, "could not determine file size for object '%s'", key) - } - size = info.Size() - } - - _, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, size, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - - if err != nil { - return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", key, bs.bucket) - } - return nil -} - -// Download retrieves a blob from the blobstore for reading. -func (bs *Blobstore) Download(key string) (io.ReadCloser, error) { - reader, err := bs.client.GetObject(context.Background(), bs.bucket, key, minio.GetObjectOptions{}) - if err != nil { - return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", key, bs.bucket) - } - return reader, nil -} - -// Delete deletes a blob from the blobstore. -func (bs *Blobstore) Delete(key string) error { - err := bs.client.RemoveObject(context.Background(), bs.bucket, key, minio.RemoveObjectOptions{}) - if err != nil { - return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", key, bs.bucket) - } - return nil -} diff --git a/pkg/storage/fs/s3ng/option.go b/pkg/storage/fs/s3ng/option.go deleted file mode 100644 index af2d9c7268..0000000000 --- a/pkg/storage/fs/s3ng/option.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng - -import ( - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - - // Endpoint of the s3 blobstore - S3Endpoint string `mapstructure:"s3.endpoint"` - - // Region of the s3 blobstore - S3Region string `mapstructure:"s3.region"` - - // Bucket of the s3 blobstore - S3Bucket string `mapstructure:"s3.bucket"` - - // Access key for the s3 blobstore - S3AccessKey string `mapstructure:"s3.access_key"` - - // Secret key for the s3 blobstore - S3SecretKey string `mapstructure:"s3.secret_key"` -} - -// S3ConfigComplete return true if all required s3 fields are set. -func (o *Options) S3ConfigComplete() bool { - return o.S3Endpoint != "" && - o.S3Region != "" && - o.S3Bucket != "" && - o.S3AccessKey != "" && - o.S3SecretKey != "" -} - -func parseConfig(m map[string]interface{}) (*Options, error) { - o := &Options{} - if err := mapstructure.Decode(m, o); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return o, nil -} diff --git a/pkg/storage/fs/s3ng/option_test.go b/pkg/storage/fs/s3ng/option_test.go deleted file mode 100644 index 46d1d60642..0000000000 --- a/pkg/storage/fs/s3ng/option_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng_test - -import ( - "github.com/cs3org/reva/pkg/storage/fs/s3ng" - "github.com/mitchellh/mapstructure" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Options", func() { - var ( - o *s3ng.Options - raw map[string]interface{} - ) - - BeforeEach(func() { - raw := map[string]interface{}{ - "s3.endpoint": "http://1.2.3.4:5000", - "s3.region": "default", - "s3.bucket": "the-bucket", - "s3.access_key": "foo", - "s3.secret_key": "bar", - } - o = &s3ng.Options{} - err := mapstructure.Decode(raw, o) - Expect(err).ToNot(HaveOccurred()) - }) - - It("parses s3 configuration", func() { - Expect(o.S3Endpoint).To(Equal("http://1.2.3.4:5000")) - Expect(o.S3Region).To(Equal("default")) - Expect(o.S3AccessKey).To(Equal("foo")) - Expect(o.S3SecretKey).To(Equal("bar")) - }) - - Describe("S3ConfigComplete", func() { - It("returns true", func() { - Expect(o.S3ConfigComplete()).To(BeTrue()) - }) - - It("returns false", func() { - fields := []string{"s3.endpoint", "s3.region", "s3.bucket", "s3.access_key", "s3.secret_key"} - for _, f := range fields { - delete(raw, f) - o = &s3ng.Options{} - err := mapstructure.Decode(raw, o) - Expect(err).ToNot(HaveOccurred()) - - Expect(o.S3ConfigComplete()).To(BeFalse(), "failed to return false on missing '%s' field", f) - } - }) - }) -}) diff --git a/pkg/storage/fs/s3ng/s3ng.go b/pkg/storage/fs/s3ng/s3ng.go deleted file mode 100644 index bda1bac61e..0000000000 --- a/pkg/storage/fs/s3ng/s3ng.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng - -import ( - "context" - "fmt" - - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/fs/s3ng/blobstore" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" -) - -func init() { - registry.Register("s3ng", New) -} - -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - o, err := parseConfig(m) - if err != nil { - return nil, err - } - - if !o.S3ConfigComplete() { - return nil, fmt.Errorf("s3 configuration incomplete") - } - - bs, err := blobstore.New(o.S3Endpoint, o.S3Region, o.S3Bucket, o.S3AccessKey, o.S3SecretKey) - if err != nil { - return nil, err - } - - return decomposedfs.NewDefault(m, bs) -} diff --git a/pkg/storage/fs/s3ng/s3ng_suite_test.go b/pkg/storage/fs/s3ng/s3ng_suite_test.go deleted file mode 100644 index 00293b99b9..0000000000 --- a/pkg/storage/fs/s3ng/s3ng_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestS3ng(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "S3ng Suite") -} diff --git a/pkg/storage/fs/s3ng/s3ng_test.go b/pkg/storage/fs/s3ng/s3ng_test.go deleted file mode 100644 index 107b25c0f7..0000000000 --- a/pkg/storage/fs/s3ng/s3ng_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng_test - -import ( - "context" - "os" - - "github.com/cs3org/reva/pkg/storage/fs/s3ng" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("S3ng", func() { - var ( - options map[string]interface{} - tmpRoot string - ) - - BeforeEach(func() { - tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") - Expect(err).ToNot(HaveOccurred()) - - options = map[string]interface{}{ - "root": tmpRoot, - "enable_home": true, - "share_folder": "/Shares", - "s3.endpoint": "http://1.2.3.4:5000", - "s3.region": "default", - "s3.bucket": "the-bucket", - "s3.access_key": "foo", - "s3.secret_key": "bar", - } - }) - - AfterEach(func() { - if tmpRoot != "" { - os.RemoveAll(tmpRoot) - } - }) - - Describe("New", func() { - It("fails on missing s3 configuration", func() { - _, err := s3ng.New(context.Background(), map[string]interface{}{}) - Expect(err).To(MatchError("s3 configuration incomplete")) - }) - - It("works with complete configuration", func() { - _, err := s3ng.New(context.Background(), options) - Expect(err).ToNot(HaveOccurred()) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go deleted file mode 100644 index b9436c76f1..0000000000 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ /dev/null @@ -1,547 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -// go:generate mockery -name PermissionsChecker -// go:generate mockery -name Tree - -import ( - "context" - "fmt" - "io" - "net/url" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "syscall" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/sharedconf" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/utils/chunking" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/storage/utils/templates" - rtrace "github.com/cs3org/reva/pkg/trace" - "github.com/cs3org/reva/pkg/utils" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// PermissionsChecker defines an interface for checking permissions on a Node. -type PermissionsChecker interface { - AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error) - HasPermission(ctx context.Context, n *node.Node, check func(*provider.ResourcePermissions) bool) (can bool, err error) -} - -// Tree is used to manage a tree hierarchy. -type Tree interface { - Setup(owner *userpb.UserId, propagateToRoot bool) error - - GetMD(ctx context.Context, node *node.Node) (os.FileInfo, error) - ListFolder(ctx context.Context, node *node.Node) ([]*node.Node, error) - // CreateHome(owner *userpb.UserId) (n *node.Node, err error) - CreateDir(ctx context.Context, node *node.Node) (err error) - // CreateReference(ctx context.Context, node *node.Node, targetURI *url.URL) error - Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error) - Delete(ctx context.Context, node *node.Node) (err error) - RestoreRecycleItemFunc(ctx context.Context, key, trashPath, restorePath string) (*node.Node, *node.Node, func() error, error) // FIXME REFERENCE use ref instead of path - PurgeRecycleItemFunc(ctx context.Context, key, purgePath string) (*node.Node, func() error, error) - - WriteBlob(key string, reader io.Reader) error - ReadBlob(key string) (io.ReadCloser, error) - DeleteBlob(key string) error - - Propagate(ctx context.Context, node *node.Node) (err error) -} - -// Decomposedfs provides the base for decomposed filesystem implementations. -type Decomposedfs struct { - lu *Lookup - tp Tree - o *options.Options - p PermissionsChecker - chunkHandler *chunking.ChunkHandler -} - -// NewDefault returns an instance with default components. -func NewDefault(m map[string]interface{}, bs tree.Blobstore) (storage.FS, error) { - o, err := options.New(m) - if err != nil { - return nil, err - } - - lu := &Lookup{} - p := node.NewPermissions(lu) - - lu.Options = o - - tp := tree.New(o.Root, o.TreeTimeAccounting, o.TreeSizeAccounting, lu, bs) - - o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr) - return New(o, lu, p, tp) -} - -// when enable home is false we want propagation to root if tree size or mtime accounting is enabled. -func enablePropagationForRoot(o *options.Options) bool { - return (!o.EnableHome && (o.TreeSizeAccounting || o.TreeTimeAccounting)) -} - -// New returns an implementation of the storage.FS interface that talks to -// a local filesystem. -func New(o *options.Options, lu *Lookup, p PermissionsChecker, tp Tree) (storage.FS, error) { - err := tp.Setup(&userpb.UserId{ - OpaqueId: o.Owner, - Idp: o.OwnerIDP, - Type: userpb.UserType(userpb.UserType_value[o.OwnerType]), - }, enablePropagationForRoot(o)) - if err != nil { - logger.New().Error().Err(err). - Msg("could not setup tree") - return nil, errors.Wrap(err, "could not setup tree") - } - - return &Decomposedfs{ - tp: tp, - lu: lu, - o: o, - p: p, - chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")), - }, nil -} - -// Shutdown shuts down the storage. -func (fs *Decomposedfs) Shutdown(ctx context.Context) error { - return nil -} - -// GetQuota returns the quota available -// TODO Document in the cs3 should we return quota or free space? -func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, err error) { - var n *node.Node - if ref != nil { - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return 0, 0, err - } - } else { - if n, err = fs.lu.HomeOrRootNode(ctx); err != nil { - return 0, 0, err - } - } - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return 0, 0, err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return 0, 0, errtypes.InternalError(err.Error()) - case !rp.GetQuota: - return 0, 0, errtypes.PermissionDenied(n.ID) - } - - ri, err := n.AsResourceInfo(ctx, &rp, []string{"treesize", "quota"}, true) - if err != nil { - return 0, 0, err - } - - quotaStr := node.QuotaUnknown - if ri.Opaque != nil && ri.Opaque.Map != nil && ri.Opaque.Map["quota"] != nil && ri.Opaque.Map["quota"].Decoder == "plain" { - quotaStr = string(ri.Opaque.Map["quota"].Value) - } - - avail, err := node.GetAvailableSize(n.InternalPath()) - if err != nil { - return 0, 0, err - } - total = avail + ri.Size - - switch { - case quotaStr == node.QuotaUncalculated, quotaStr == node.QuotaUnknown, quotaStr == node.QuotaUnlimited: - // best we can do is return current total - // TODO indicate unlimited total? -> in opaque data? - default: - if quota, err := strconv.ParseUint(quotaStr, 10, 64); err == nil { - if total > quota { - total = quota - } - } - } - - return total, ri.Size, nil -} - -// CreateHome creates a new home node for the given user. -func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { - if !fs.o.EnableHome || fs.o.UserLayout == "" { - return errtypes.NotSupported("Decomposedfs: CreateHome() home supported disabled") - } - - var n, h *node.Node - if n, err = fs.lu.RootNode(ctx); err != nil { - return - } - h, err = fs.lu.WalkPath(ctx, n, fs.lu.mustGetUserLayout(ctx), false, func(ctx context.Context, n *node.Node) error { - if !n.Exists { - if err := fs.tp.CreateDir(ctx, n); err != nil { - return err - } - } - return nil - }) - if err != nil { - return - } - - // update the owner - u := ctxpkg.ContextMustGetUser(ctx) - if err = h.WriteMetadata(u.Id); err != nil { - return - } - - if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - homePath := h.InternalPath() - // mark the home node as the end of propagation - if err = xattr.Set(homePath, xattrs.PropagationAttr, []byte("1")); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", h).Msg("could not mark home as propagation root") - return - } - } - - if err := h.SetMetadata(xattrs.SpaceNameAttr, u.DisplayName); err != nil { - return err - } - - // add storage space - if err := fs.createStorageSpace(ctx, "personal", h.ID); err != nil { - return err - } - - return -} - -// The os not exists error is buried inside the xattr error, -// so we cannot just use os.IsNotExists(). -func isAlreadyExists(err error) bool { - if xerr, ok := err.(*os.LinkError); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.EEXIST - } - } - return false -} - -// GetHome is called to look up the home path for a user -// It is NOT supposed to return the internal path but the external path. -func (fs *Decomposedfs) GetHome(ctx context.Context) (string, error) { - if !fs.o.EnableHome || fs.o.UserLayout == "" { - return "", errtypes.NotSupported("Decomposedfs: GetHome() home supported disabled") - } - u := ctxpkg.ContextMustGetUser(ctx) - layout := templates.WithUser(u, fs.o.UserLayout) - return filepath.Join(fs.o.Root, layout), nil // TODO use a namespace? -} - -// GetPathByID returns the fn pointed by the file id, without the internal namespace. -func (fs *Decomposedfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { - node, err := fs.lu.NodeFromID(ctx, id) - if err != nil { - return "", err - } - - return fs.lu.Path(ctx, node) -} - -// CreateDir creates the specified directory. -func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) (err error) { - name := path.Base(ref.Path) - if name == "" || name == "." || name == "/" { - return errtypes.BadRequest("Invalid path") - } - ref.Path = path.Dir(ref.Path) - var n *node.Node - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if n, err = n.Child(ctx, name); err != nil { - return - } - - if n.Exists { - return errtypes.AlreadyExists(ref.Path) - } - pn, err := n.Parent() - if err != nil { - return errors.Wrap(err, "decomposedfs: error getting parent "+n.ParentID) - } - ok, err := fs.p.HasPermission(ctx, pn, func(rp *provider.ResourcePermissions) bool { - return rp.CreateContainer - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - err = fs.tp.CreateDir(ctx, n) - - if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - nodePath := n.InternalPath() - // mark the home node as the end of propagation - if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") - return - } - } - return -} - -// TouchFile as defined in the storage.FS interface. -func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference) error { - return fmt.Errorf("unimplemented: TouchFile") -} - -// CreateReference creates a reference as a node folder with the target stored in extended attributes -// There is no difference between the /Shares folder and normal nodes because the storage is not supposed to be accessible without the storage provider. -// In effect everything is a shadow namespace. -// To mimic the eos end owncloud driver we only allow references as children of the "/Shares" folder -// TODO when home support is enabled should the "/Shares" folder still be listed? -func (fs *Decomposedfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) (err error) { - p = strings.Trim(p, "/") - parts := strings.Split(p, "/") - - if len(parts) != 2 { - return errtypes.PermissionDenied("Decomposedfs: references must be a child of the share folder: share_folder=" + fs.o.ShareFolder + " path=" + p) - } - - if parts[0] != strings.Trim(fs.o.ShareFolder, "/") { - return errtypes.PermissionDenied("Decomposedfs: cannot create references outside the share folder: share_folder=" + fs.o.ShareFolder + " path=" + p) - } - - // create Shares folder if it does not exist - var n *node.Node - if n, err = fs.lu.NodeFromPath(ctx, fs.o.ShareFolder, false); err != nil { - return errtypes.InternalError(err.Error()) - } else if !n.Exists { - if err = fs.tp.CreateDir(ctx, n); err != nil { - return - } - } - - if n, err = n.Child(ctx, parts[1]); err != nil { - return errtypes.InternalError(err.Error()) - } - - if n.Exists { - // TODO append increasing number to mountpoint name - return errtypes.AlreadyExists(p) - } - - if err = fs.tp.CreateDir(ctx, n); err != nil { - return - } - - internal := n.InternalPath() - if err = xattr.Set(internal, xattrs.ReferenceAttr, []byte(targetURI.String())); err != nil { - return errors.Wrapf(err, "Decomposedfs: error setting the target %s on the reference file %s", targetURI.String(), internal) - } - return nil -} - -// Move moves a resource from one reference to another. -func (fs *Decomposedfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { - var oldNode, newNode *node.Node - if oldNode, err = fs.lu.NodeFromResource(ctx, oldRef); err != nil { - return - } - - if !oldNode.Exists { - err = errtypes.NotFound(filepath.Join(oldNode.ParentID, oldNode.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, oldNode, func(rp *provider.ResourcePermissions) bool { - return rp.Move - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(oldNode.ID) - } - - if newNode, err = fs.lu.NodeFromResource(ctx, newRef); err != nil { - return - } - if newNode.Exists { - err = errtypes.AlreadyExists(filepath.Join(newNode.ParentID, newNode.Name)) - return - } - - return fs.tp.Move(ctx, oldNode, newNode) -} - -// GetMD returns the metadata for the specified resource. -func (fs *Decomposedfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !rp.Stat: - return nil, errtypes.PermissionDenied(node.ID) - } - - return node.AsResourceInfo(ctx, &rp, mdKeys, utils.IsRelativeReference(ref)) -} - -// ListFolder returns a list of resources in the specified folder. -func (fs *Decomposedfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { - var n *node.Node - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - - ctx, span := rtrace.Provider.Tracer("decomposedfs").Start(ctx, "ListFolder") - defer span.End() - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !rp.ListContainer: - return nil, errtypes.PermissionDenied(n.ID) - } - - var children []*node.Node - children, err = fs.tp.ListFolder(ctx, n) - if err != nil { - return - } - - for i := range children { - np := rp - // add this childs permissions - pset := n.PermissionSet(ctx) - node.AddPermissions(&np, &pset) - if ri, err := children[i].AsResourceInfo(ctx, &np, mdKeys, utils.IsRelativeReference(ref)); err == nil { - finfos = append(finfos, ri) - } - } - return -} - -// Delete deletes the specified resource. -func (fs *Decomposedfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { - return rp.Delete - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) - } - - return fs.tp.Delete(ctx, node) -} - -// Download returns a reader to the specified resource. -func (fs *Decomposedfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - node, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: error resolving ref") - } - - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return nil, err - } - - ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { - return rp.InitiateFileDownload - }) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !ok: - return nil, errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) - } - - reader, err := fs.tp.ReadBlob(node.BlobID) - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: error download blob '"+node.ID+"'") - } - return reader, nil -} - -// GetLock returns an existing lock on the given reference. -func (fs *Decomposedfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { - return nil, errtypes.NotSupported("unimplemented") -} - -// SetLock puts a lock on the given reference. -func (fs *Decomposedfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - return errtypes.NotSupported("unimplemented") -} - -// RefreshLock refreshes an existing lock on the given reference. -func (fs *Decomposedfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock, existingLockID string) error { - return errtypes.NotSupported("unimplemented") -} - -// Unlock removes an existing lock from the given reference. -func (fs *Decomposedfs) Unlock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - return errtypes.NotSupported("unimplemented") -} diff --git a/pkg/storage/utils/decomposedfs/decomposedfs_concurrency_test.go b/pkg/storage/utils/decomposedfs/decomposedfs_concurrency_test.go deleted file mode 100644 index ccb92d145f..0000000000 --- a/pkg/storage/utils/decomposedfs/decomposedfs_concurrency_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - "context" - "os" - "path" - "sync" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" - treemocks "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree/mocks" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Decomposed", func() { - var ( - options map[string]interface{} - ctx context.Context - tmpRoot string - fs storage.FS - ) - - BeforeEach(func() { - tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") - Expect(err).ToNot(HaveOccurred()) - - options = map[string]interface{}{ - "root": tmpRoot, - "share_folder": "/Shares", - "enable_home": false, - "user_layout": "{{.Id.OpaqueId}}", - "owner": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - } - u := &userpb.User{ - Id: &userpb.UserId{ - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - }, - Username: "test", - Mail: "marie@example.org", - DisplayName: "Marie Curie", - Groups: []string{ - "radium-lovers", - "polonium-lovers", - "physics-lovers", - }, - } - ctx = ctxpkg.ContextSetUser(context.Background(), u) - - bs := &treemocks.Blobstore{} - fs, err = decomposedfs.NewDefault(options, bs) - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - if tmpRoot != "" { - os.RemoveAll(tmpRoot) - } - }) - - Describe("concurrent", func() { - Describe("Upload", func() { - var ( - r1 = []byte("test") - r2 = []byte("another run") - ) - - PIt("generates two revisions", func() { - // runtime.GOMAXPROCS(1) // uncomment to remove concurrency and see revisions working. - wg := &sync.WaitGroup{} - wg.Add(2) - - // upload file with contents: "test" - go func(wg *sync.WaitGroup) { - _ = helpers.Upload(ctx, fs, &provider.Reference{Path: "uploaded.txt"}, r1) - wg.Done() - }(wg) - - // upload file with contents: "another run" - go func(wg *sync.WaitGroup) { - _ = helpers.Upload(ctx, fs, &provider.Reference{Path: "uploaded.txt"}, r2) - wg.Done() - }(wg) - - // this test, by the way the oCIS storage is implemented, is non-deterministic, and the contents - // of uploaded.txt will change on each run depending on which of the 2 routines above makes it - // first into the scheduler. In order to make it deterministic, we have to consider the Upload impl- - // ementation and we can leverage concurrency and add locks only when the destination path are the - // same for 2 uploads. - - wg.Wait() - revisions, err := fs.ListRevisions(ctx, &provider.Reference{Path: "uploaded.txt"}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(revisions)).To(Equal(1)) - - _, err = os.ReadFile(path.Join(tmpRoot, "nodes", "root", "uploaded.txt")) - Expect(err).ToNot(HaveOccurred()) - }) - }) - - Describe("CreateDir", func() { - It("handle already existing directories", func() { - for i := 0; i < 10; i++ { - go func() { - defer GinkgoRecover() - err := fs.CreateDir(ctx, &provider.Reference{Path: "/fightforit"}) - if err != nil { - rinfo, err := fs.GetMD(ctx, &provider.Reference{Path: "/fightforit"}, nil) - Expect(err).ToNot(HaveOccurred()) - Expect(rinfo).ToNot(BeNil()) - } - }() - } - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs_suite_test.go b/pkg/storage/utils/decomposedfs/decomposedfs_suite_test.go deleted file mode 100644 index 17172c441c..0000000000 --- a/pkg/storage/utils/decomposedfs/decomposedfs_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestDecomposed(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Decomposed Suite") -} diff --git a/pkg/storage/utils/decomposedfs/decomposedfs_test.go b/pkg/storage/utils/decomposedfs/decomposedfs_test.go deleted file mode 100644 index f9be4a88be..0000000000 --- a/pkg/storage/utils/decomposedfs/decomposedfs_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" - helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" - treemocks "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree/mocks" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/stretchr/testify/mock" -) - -var _ = Describe("Decomposed", func() { - var ( - env *helpers.TestEnv - - ref *provider.Reference - ) - - BeforeEach(func() { - ref = &provider.Reference{Path: "/dir1"} - }) - - JustBeforeEach(func() { - var err error - env, err = helpers.NewTestEnv() - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - if env != nil { - env.Cleanup() - } - }) - - Describe("NewDefault", func() { - It("works", func() { - bs := &treemocks.Blobstore{} - _, err := decomposedfs.NewDefault(map[string]interface{}{ - "root": env.Root, - }, bs) - Expect(err).ToNot(HaveOccurred()) - }) - }) - - Describe("Delete", func() { - Context("with insufficient permissions", func() { - It("returns an error", func() { - env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - - err := env.Fs.Delete(env.Ctx, ref) - - Expect(err).To(MatchError(ContainSubstring("permission denied"))) - }) - }) - - Context("with sufficient permissions", func() { - JustBeforeEach(func() { - env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - }) - - It("does not (yet) delete the blob from the blobstore", func() { - err := env.Fs.Delete(env.Ctx, ref) - - Expect(err).ToNot(HaveOccurred()) - env.Blobstore.AssertNotCalled(GinkgoT(), "Delete", mock.AnythingOfType("string")) - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go deleted file mode 100644 index bf61e3b6d9..0000000000 --- a/pkg/storage/utils/decomposedfs/grants.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "path/filepath" - "strings" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/ace" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/pkg/xattr" -) - -// SpaceGrant is the key used to signal not to create a new space when a grant is assigned to a storage space. -var SpaceGrant struct{} - -// DenyGrant denies access to a resource. -func (fs *Decomposedfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { - return errtypes.NotSupported("decomposedfs: not supported") -} - -// AddGrant adds a grant to a resource. -func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - log := appctx.GetLogger(ctx) - log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { - // TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92 - return rp.AddGrant || rp.UpdateGrant - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) - } - - np := fs.lu.InternalPath(node.ID) - e := ace.FromGrant(g) - principal, value := e.Marshal() - if err := xattr.Set(np, xattrs.GrantPrefix+principal, value); err != nil { - return err - } - - // when a grant is added to a space, do not add a new space under "shares" - if spaceGrant := ctx.Value(SpaceGrant); spaceGrant == nil { - err := fs.createStorageSpace(ctx, "share", node.ID) - if err != nil { - return err - } - } - - return fs.tp.Propagate(ctx, node) -} - -// ListGrants lists the grants on the specified resource. -func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { - return rp.ListGrants - }) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !ok: - return nil, errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) - } - - log := appctx.GetLogger(ctx) - np := fs.lu.InternalPath(node.ID) - var attrs []string - if attrs, err = xattr.List(np); err != nil { - log.Error().Err(err).Msg("error listing attributes") - return nil, err - } - - log.Debug().Interface("attrs", attrs).Msg("read attributes") - - aces := extractACEsFromAttrs(ctx, np, attrs) - - grants = make([]*provider.Grant, 0, len(aces)) - for i := range aces { - grants = append(grants, aces[i].Grant()) - } - - return grants, nil -} - -// RemoveGrant removes a grant from resource. -func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { - return rp.RemoveGrant - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) - } - - var attr string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - attr = xattrs.GrantPrefix + xattrs.GroupAcePrefix + g.Grantee.GetGroupId().OpaqueId - } else { - attr = xattrs.GrantPrefix + xattrs.UserAcePrefix + g.Grantee.GetUserId().OpaqueId - } - - np := fs.lu.InternalPath(node.ID) - if err = xattr.Remove(np, attr); err != nil { - return - } - - return fs.tp.Propagate(ctx, node) -} - -// UpdateGrant updates a grant on a resource. -func (fs *Decomposedfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - // TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92 - return fs.AddGrant(ctx, ref, g) -} - -// extractACEsFromAttrs reads ACEs in the list of attrs from the node. -func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { - log := appctx.GetLogger(ctx) - entries = []*ace.ACE{} - for i := range attrs { - if strings.HasPrefix(attrs[i], xattrs.GrantPrefix) { - var value []byte - var err error - if value, err = xattr.Get(fsfn, attrs[i]); err != nil { - log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") - continue - } - var e *ace.ACE - principal := attrs[i][len(xattrs.GrantPrefix):] - if e, err = ace.Unmarshal(principal, value); err != nil { - log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace") - continue - } - entries = append(entries, e) - } - } - return -} diff --git a/pkg/storage/utils/decomposedfs/grants_test.go b/pkg/storage/utils/decomposedfs/grants_test.go deleted file mode 100644 index ace54e7730..0000000000 --- a/pkg/storage/utils/decomposedfs/grants_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - "io/fs" - "os" - "path" - "path/filepath" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/xattr" - "github.com/stretchr/testify/mock" -) - -type testFS struct { - root string -} - -func (t testFS) Open(name string) (fs.File, error) { - return os.Open(filepath.Join(t.root, name)) -} - -var _ = Describe("Grants", func() { - var ( - env *helpers.TestEnv - ref *provider.Reference - grant *provider.Grant - tfs = &testFS{} - ) - - BeforeEach(func() { - ref = &provider.Reference{Path: "/dir1"} - - grant = &provider.Grant{ - Grantee: &provider.Grantee{ - Type: provider.GranteeType_GRANTEE_TYPE_USER, - Id: &provider.Grantee_UserId{ - UserId: &userpb.UserId{ - OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51", - }, - }, - }, - Permissions: &provider.ResourcePermissions{ - Stat: true, - Move: true, - Delete: false, - }, - } - }) - - JustBeforeEach(func() { - var err error - env, err = helpers.NewTestEnv() - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - if env != nil { - env.Cleanup() - } - }) - - Context("with insufficient permissions", func() { - JustBeforeEach(func() { - env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - }) - - Describe("AddGrant", func() { - It("adds grants", func() { - err := env.Fs.AddGrant(env.Ctx, ref, grant) - Expect(err).To(MatchError(ContainSubstring("permission denied"))) - }) - }) - }) - - Context("with sufficient permissions", func() { - JustBeforeEach(func() { - env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - }) - - Describe("AddGrant", func() { - It("adds grants", func() { - n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1", false) - Expect(err).ToNot(HaveOccurred()) - - err = env.Fs.AddGrant(env.Ctx, ref, grant) - Expect(err).ToNot(HaveOccurred()) - - localPath := path.Join(env.Root, "nodes", n.ID) - attr, err := xattr.Get(localPath, xattrs.GrantPrefix+xattrs.UserAcePrefix+grant.Grantee.GetUserId().OpaqueId) - Expect(err).ToNot(HaveOccurred()) - Expect(string(attr)).To(Equal("\x00t=A:f=:p=rw")) - }) - - It("creates a storage space per created grant", func() { - err := env.Fs.AddGrant(env.Ctx, ref, grant) - Expect(err).ToNot(HaveOccurred()) - - spacesPath := filepath.Join(env.Root, "spaces") - tfs.root = spacesPath - entries, err := fs.ReadDir(tfs, "share") - Expect(err).ToNot(HaveOccurred()) - Expect(len(entries)).To(BeNumerically(">=", 1)) - }) - }) - - Describe("ListGrants", func() { - It("lists existing grants", func() { - err := env.Fs.AddGrant(env.Ctx, ref, grant) - Expect(err).ToNot(HaveOccurred()) - - grants, err := env.Fs.ListGrants(env.Ctx, ref) - Expect(err).ToNot(HaveOccurred()) - Expect(len(grants)).To(Equal(1)) - - g := grants[0] - Expect(g.Grantee.GetUserId().OpaqueId).To(Equal(grant.Grantee.GetUserId().OpaqueId)) - Expect(g.Permissions.Stat).To(BeTrue()) - Expect(g.Permissions.Move).To(BeTrue()) - Expect(g.Permissions.Delete).To(BeFalse()) - }) - }) - - Describe("RemoveGrants", func() { - It("removes the grant", func() { - err := env.Fs.AddGrant(env.Ctx, ref, grant) - Expect(err).ToNot(HaveOccurred()) - - grants, err := env.Fs.ListGrants(env.Ctx, ref) - Expect(err).ToNot(HaveOccurred()) - Expect(len(grants)).To(Equal(1)) - - err = env.Fs.RemoveGrant(env.Ctx, ref, grant) - Expect(err).ToNot(HaveOccurred()) - - grants, err = env.Fs.ListGrants(env.Ctx, ref) - Expect(err).ToNot(HaveOccurred()) - Expect(len(grants)).To(Equal(0)) - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go deleted file mode 100644 index acb7c7fa92..0000000000 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/pkg/xattr" -) - -// Lookup implements transformations from filepath to node and back. -type Lookup struct { - Options *options.Options -} - -// NodeFromResource takes in a request path or request id and converts it to a Node. -func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) (*node.Node, error) { - if ref.ResourceId != nil { - // check if a storage space reference is used - // currently, the decomposed fs uses the root node id as the space id - n, err := lu.NodeFromID(ctx, ref.ResourceId) - if err != nil { - return nil, err - } - // is this a relative reference? - if ref.Path != "" { - p := filepath.Clean(ref.Path) - if p != "." { - // walk the relative path - n, err = lu.WalkPath(ctx, n, p, false, func(ctx context.Context, n *node.Node) error { - return nil - }) - if err != nil { - return nil, err - } - } - } - return n, nil - } - - if ref.Path != "" { - return lu.NodeFromPath(ctx, ref.GetPath(), false) - } - - // reference is invalid - return nil, fmt.Errorf("invalid reference %+v. at least resource_id or path must be set", ref) -} - -// NodeFromPath converts a filename into a Node. -func (lu *Lookup) NodeFromPath(ctx context.Context, fn string, followReferences bool) (*node.Node, error) { - log := appctx.GetLogger(ctx) - log.Debug().Interface("fn", fn).Msg("NodeFromPath()") - - root, err := lu.HomeOrRootNode(ctx) - if err != nil { - return nil, err - } - - n := root - // TODO collect permissions of the current user on every segment - fn = filepath.Clean(fn) - if fn != "/" && fn != "." { - n, err = lu.WalkPath(ctx, n, fn, followReferences, func(ctx context.Context, n *node.Node) error { - log.Debug().Interface("node", n).Msg("NodeFromPath() walk") - if n.SpaceRoot != nil && n.SpaceRoot != root { - root = n.SpaceRoot - } - return nil - }) - if err != nil { - return nil, err - } - } - n.SpaceRoot = root - return n, nil -} - -// NodeFromID returns the internal path for the id. -func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *node.Node, err error) { - if id == nil || id.OpaqueId == "" { - return nil, fmt.Errorf("invalid resource id %+v", id) - } - n, err = node.ReadNode(ctx, lu, id.OpaqueId) - if err != nil { - return nil, err - } - - return n, n.FindStorageSpaceRoot() -} - -// Path returns the path for node. -func (lu *Lookup) Path(ctx context.Context, n *node.Node) (p string, err error) { - var root *node.Node - if root, err = lu.HomeOrRootNode(ctx); err != nil { - return - } - for n.ID != root.ID { - p = filepath.Join(n.Name, p) - if n, err = n.Parent(); err != nil { - appctx.GetLogger(ctx). - Error().Err(err). - Str("path", p). - Interface("node", n). - Msg("Path()") - return - } - } - p = filepath.Join("/", p) - return -} - -// RootNode returns the root node of the storage. -func (lu *Lookup) RootNode(ctx context.Context) (*node.Node, error) { - n := node.New("root", "", "", 0, "", nil, lu) - n.Exists = true - return n, nil -} - -// HomeNode returns the home node of a user. -func (lu *Lookup) HomeNode(ctx context.Context) (node *node.Node, err error) { - if !lu.Options.EnableHome { - return nil, errtypes.NotSupported("Decomposedfs: home supported disabled") - } - - if node, err = lu.RootNode(ctx); err != nil { - return - } - node, err = lu.WalkPath(ctx, node, lu.mustGetUserLayout(ctx), false, nil) - return -} - -// WalkPath calls n.Child(segment) on every path segment in p starting at the node r. -// If a function f is given it will be executed for every segment node, but not the root node r. -// If followReferences is given the current visited reference node is replaced by the referenced node. -func (lu *Lookup) WalkPath(ctx context.Context, r *node.Node, p string, followReferences bool, f func(ctx context.Context, n *node.Node) error) (*node.Node, error) { - segments := strings.Split(strings.Trim(p, "/"), "/") - var err error - for i := range segments { - if r, err = r.Child(ctx, segments[i]); err != nil { - return r, err - } - - if followReferences { - if attrBytes, err := xattr.Get(r.InternalPath(), xattrs.ReferenceAttr); err == nil { - realNodeID := attrBytes - ref, err := xattrs.ReferenceFromAttr(realNodeID) - if err != nil { - return nil, err - } - - r, err = lu.NodeFromID(ctx, ref.ResourceId) - if err != nil { - return nil, err - } - } - } - if node.IsSpaceRoot(r) { - r.SpaceRoot = r - } - - if !r.Exists && i < len(segments)-1 { - return r, errtypes.NotFound(segments[i]) - } - if f != nil { - if err = f(ctx, r); err != nil { - return r, err - } - } - } - return r, nil -} - -// HomeOrRootNode returns the users home node when home support is enabled. -// it returns the storages root node otherwise. -func (lu *Lookup) HomeOrRootNode(ctx context.Context) (node *node.Node, err error) { - if lu.Options.EnableHome { - return lu.HomeNode(ctx) - } - return lu.RootNode(ctx) -} - -// InternalRoot returns the internal storage root directory. -func (lu *Lookup) InternalRoot() string { - return lu.Options.Root -} - -// InternalPath returns the internal path for a given ID. -func (lu *Lookup) InternalPath(id string) string { - return filepath.Join(lu.Options.Root, "nodes", id) -} - -func (lu *Lookup) mustGetUserLayout(ctx context.Context) string { - u := ctxpkg.ContextMustGetUser(ctx) - return templates.WithUser(u, lu.Options.UserLayout) -} - -// ShareFolder returns the internal storage root directory. -func (lu *Lookup) ShareFolder() string { - return lu.Options.ShareFolder -} diff --git a/pkg/storage/utils/decomposedfs/lookup_test.go b/pkg/storage/utils/decomposedfs/lookup_test.go deleted file mode 100644 index 1c9df693ce..0000000000 --- a/pkg/storage/utils/decomposedfs/lookup_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Lookup", func() { - var ( - env *helpers.TestEnv - ) - - JustBeforeEach(func() { - var err error - env, err = helpers.NewTestEnv() - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - if env != nil { - env.Cleanup() - } - }) - - Describe("Node from path", func() { - It("returns the path including a leading slash", func() { - n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - - path, err := env.Lookup.Path(env.Ctx, n) - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal("/dir1/file1")) - }) - }) - - Describe("Node From Resource only by path", func() { - It("returns the path including a leading slash and the space root is set", func() { - n, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{Path: "/dir1/subdir1/file2"}) - Expect(err).ToNot(HaveOccurred()) - - path, err := env.Lookup.Path(env.Ctx, n) - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal("/dir1/subdir1/file2")) - Expect(n.SpaceRoot.Name).To(Equal("userid")) - Expect(n.SpaceRoot.ParentID).To(Equal("root")) - }) - }) - - Describe("Node From Resource only by id", func() { - It("returns the path including a leading slash and the space root is set", func() { - // do a node lookup by path - nRef, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - - // try to find the same node by id - n, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ResourceId: &provider.ResourceId{OpaqueId: nRef.ID}}) - Expect(err).ToNot(HaveOccurred()) - - // Check if we got the right node and spaceRoot - path, err := env.Lookup.Path(env.Ctx, n) - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal("/dir1/file1")) - Expect(n.SpaceRoot.Name).To(Equal("userid")) - Expect(n.SpaceRoot.ParentID).To(Equal("root")) - }) - }) - - Describe("Node From Resource by id and relative path", func() { - It("returns the path including a leading slash and the space root is set", func() { - // do a node lookup by path for the parent - nRef, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1", false) - Expect(err).ToNot(HaveOccurred()) - - // try to find the child node by parent id and relative path - n, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ResourceId: &provider.ResourceId{OpaqueId: nRef.ID}, Path: "./file1"}) - Expect(err).ToNot(HaveOccurred()) - - // Check if we got the right node and spaceRoot - path, err := env.Lookup.Path(env.Ctx, n) - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal("/dir1/file1")) - Expect(n.SpaceRoot.Name).To(Equal("userid")) - Expect(n.SpaceRoot.ParentID).To(Equal("root")) - }) - }) - - Describe("Reference Parsing", func() { - It("parses a valid cs3 reference", func() { - in := []byte("cs3:bede11a0-ea3d-11eb-a78b-bf907adce8ed/c402d01c-ea3d-11eb-a0fc-c32f9d32528f") - ref, err := xattrs.ReferenceFromAttr(in) - - Expect(err).ToNot(HaveOccurred()) - Expect(ref.ResourceId.StorageId).To(Equal("bede11a0-ea3d-11eb-a78b-bf907adce8ed")) - Expect(ref.ResourceId.OpaqueId).To(Equal("c402d01c-ea3d-11eb-a0fc-c32f9d32528f")) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/metadata.go b/pkg/storage/utils/decomposedfs/metadata.go deleted file mode 100644 index be5d52a92d..0000000000 --- a/pkg/storage/utils/decomposedfs/metadata.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "path/filepath" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/utils" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// SetArbitraryMetadata sets the metadata on a resource. -func (fs *Decomposedfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "decomposedfs: error resolving ref") - } - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - // TODO add explicit SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 - return rp.InitiateFileUpload - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - nodePath := n.InternalPath() - - errs := []error{} - // TODO should we really continue updating when an error occurs? - if md.Metadata != nil { - if val, ok := md.Metadata["mtime"]; ok { - delete(md.Metadata, "mtime") - err := n.SetMtime(ctx, val) - if err != nil { - errs = append(errs, errors.Wrap(err, "could not set mtime")) - } - } - // TODO(jfd) special handling for atime? - // TODO(jfd) allow setting birth time (btime)? - // TODO(jfd) any other metadata that is interesting? fileid? - // TODO unset when file is updated - // TODO unset when folder is updated or add timestamp to etag? - if val, ok := md.Metadata["etag"]; ok { - delete(md.Metadata, "etag") - err := n.SetEtag(ctx, val) - if err != nil { - errs = append(errs, errors.Wrap(err, "could not set etag")) - } - } - if val, ok := md.Metadata[node.FavoriteKey]; ok { - delete(md.Metadata, node.FavoriteKey) - if u, ok := ctxpkg.ContextGetUser(ctx); ok { - if uid := u.GetId(); uid != nil { - if err := n.SetFavorite(uid, val); err != nil { - sublog.Error().Err(err). - Interface("user", u). - Msg("could not set favorite flag") - errs = append(errs, errors.Wrap(err, "could not set favorite flag")) - } - } else { - sublog.Error().Interface("user", u).Msg("user has no id") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) - } - } else { - sublog.Error().Interface("user", u).Msg("error getting user from ctx") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) - } - } - } - for k, v := range md.Metadata { - attrName := xattrs.MetadataPrefix + k - if err = xattr.Set(nodePath, attrName, []byte(v)); err != nil { - errs = append(errs, errors.Wrap(err, "decomposedfs: could not set metadata attribute "+attrName+" to "+k)) - } - } - - switch len(errs) { - case 0: - return fs.tp.Propagate(ctx, n) - case 1: - // TODO Propagate if anything changed - return errs[0] - default: - // TODO Propagate if anything changed - // TODO how to return multiple errors? - return errors.New("multiple errors occurred, see log for details") - } -} - -// UnsetArbitraryMetadata unsets the metadata on the given resource. -func (fs *Decomposedfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "decomposedfs: error resolving ref") - } - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - // TODO use SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 - return rp.InitiateFileUpload - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - nodePath := n.InternalPath() - errs := []error{} - for _, k := range keys { - switch k { - case node.FavoriteKey: - if u, ok := ctxpkg.ContextGetUser(ctx); ok { - // the favorite flag is specific to the user, so we need to incorporate the userid - if uid := u.GetId(); uid != nil { - fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if err := xattr.Remove(nodePath, fa); err != nil { - sublog.Error().Err(err). - Interface("user", u). - Str("key", fa). - Msg("could not unset favorite flag") - errs = append(errs, errors.Wrap(err, "could not unset favorite flag")) - } - } else { - sublog.Error(). - Interface("user", u). - Msg("user has no id") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) - } - } else { - sublog.Error(). - Interface("user", u). - Msg("error getting user from ctx") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) - } - default: - if err = xattr.Remove(nodePath, xattrs.MetadataPrefix+k); err != nil { - // a non-existing attribute will return an error, which we can ignore - // (using string compare because the error type is syscall.Errno and not wrapped/recognizable) - if e, ok := err.(*xattr.Error); !ok || !(e.Err.Error() == "no data available" || - // darwin - e.Err.Error() == "attribute not found") { - sublog.Error().Err(err). - Str("key", k). - Msg("could not unset metadata") - errs = append(errs, errors.Wrap(err, "could not unset metadata")) - } - } - } - } - switch len(errs) { - case 0: - return fs.tp.Propagate(ctx, n) - case 1: - // TODO Propagate if anything changed - return errs[0] - default: - // TODO Propagate if anything changed - // TODO how to return multiple errors? - return errors.New("multiple errors occurred, see log for details") - } -} diff --git a/pkg/storage/utils/decomposedfs/mocks/PermissionsChecker.go b/pkg/storage/utils/decomposedfs/mocks/PermissionsChecker.go deleted file mode 100644 index 54e078220b..0000000000 --- a/pkg/storage/utils/decomposedfs/mocks/PermissionsChecker.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - - node "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - mock "github.com/stretchr/testify/mock" - - providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" -) - -// PermissionsChecker is an autogenerated mock type for the PermissionsChecker type -type PermissionsChecker struct { - mock.Mock -} - -// AssemblePermissions provides a mock function with given fields: ctx, n -func (_m *PermissionsChecker) AssemblePermissions(ctx context.Context, n *node.Node) (providerv1beta1.ResourcePermissions, error) { - ret := _m.Called(ctx, n) - - var r0 providerv1beta1.ResourcePermissions - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) providerv1beta1.ResourcePermissions); ok { - r0 = rf(ctx, n) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(providerv1beta1.ResourcePermissions) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *node.Node) error); ok { - r1 = rf(ctx, n) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// HasPermission provides a mock function with given fields: ctx, n, check -func (_m *PermissionsChecker) HasPermission(ctx context.Context, n *node.Node, check func(*providerv1beta1.ResourcePermissions) bool) (bool, error) { - ret := _m.Called(ctx, n, check) - - var r0 bool - if rf, ok := ret.Get(0).(func(context.Context, *node.Node, func(*providerv1beta1.ResourcePermissions) bool) bool); ok { - r0 = rf(ctx, n, check) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *node.Node, func(*providerv1beta1.ResourcePermissions) bool) error); ok { - r1 = rf(ctx, n, check) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/pkg/storage/utils/decomposedfs/mocks/Tree.go b/pkg/storage/utils/decomposedfs/mocks/Tree.go deleted file mode 100644 index 51acf3190a..0000000000 --- a/pkg/storage/utils/decomposedfs/mocks/Tree.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - io "io" - - mock "github.com/stretchr/testify/mock" - - node "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - - os "os" - - providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" -) - -// Tree is an autogenerated mock type for the Tree type -type Tree struct { - mock.Mock -} - -// CreateDir provides a mock function with given fields: ctx, _a1 -func (_m *Tree) CreateDir(ctx context.Context, _a1 *node.Node) error { - ret := _m.Called(ctx, _a1) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) error); ok { - r0 = rf(ctx, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Delete provides a mock function with given fields: ctx, _a1 -func (_m *Tree) Delete(ctx context.Context, _a1 *node.Node) error { - ret := _m.Called(ctx, _a1) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) error); ok { - r0 = rf(ctx, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteBlob provides a mock function with given fields: key -func (_m *Tree) DeleteBlob(key string) error { - ret := _m.Called(key) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(key) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetMD provides a mock function with given fields: ctx, _a1 -func (_m *Tree) GetMD(ctx context.Context, _a1 *node.Node) (os.FileInfo, error) { - ret := _m.Called(ctx, _a1) - - var r0 os.FileInfo - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) os.FileInfo); ok { - r0 = rf(ctx, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(os.FileInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *node.Node) error); ok { - r1 = rf(ctx, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetPathByID provides a mock function with given fields: ctx, id -func (_m *Tree) GetPathByID(ctx context.Context, id *providerv1beta1.Reference) (string, error) { - ret := _m.Called(ctx, id) - - var r0 string - if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.Reference) string); ok { - r0 = rf(ctx, id) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.Reference) error); ok { - r1 = rf(ctx, id) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListFolder provides a mock function with given fields: ctx, _a1 -func (_m *Tree) ListFolder(ctx context.Context, _a1 *node.Node) ([]*node.Node, error) { - ret := _m.Called(ctx, _a1) - - var r0 []*node.Node - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) []*node.Node); ok { - r0 = rf(ctx, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*node.Node) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *node.Node) error); ok { - r1 = rf(ctx, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Move provides a mock function with given fields: ctx, oldNode, newNode -func (_m *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) error { - ret := _m.Called(ctx, oldNode, newNode) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *node.Node, *node.Node) error); ok { - r0 = rf(ctx, oldNode, newNode) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Propagate provides a mock function with given fields: ctx, _a1 -func (_m *Tree) Propagate(ctx context.Context, _a1 *node.Node) error { - ret := _m.Called(ctx, _a1) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *node.Node) error); ok { - r0 = rf(ctx, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// PurgeRecycleItemFunc provides a mock function with given fields: ctx, key -func (_m *Tree) PurgeRecycleItemFunc(ctx context.Context, key string) (*node.Node, func() error, error) { - ret := _m.Called(ctx, key) - - var r0 *node.Node - if rf, ok := ret.Get(0).(func(context.Context, string) *node.Node); ok { - r0 = rf(ctx, key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*node.Node) - } - } - - var r1 func() error - if rf, ok := ret.Get(1).(func(context.Context, string) func() error); ok { - r1 = rf(ctx, key) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(func() error) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context, string) error); ok { - r2 = rf(ctx, key) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// ReadBlob provides a mock function with given fields: key -func (_m *Tree) ReadBlob(key string) (io.ReadCloser, error) { - ret := _m.Called(key) - - var r0 io.ReadCloser - if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { - r0 = rf(key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.ReadCloser) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(key) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RestoreRecycleItemFunc provides a mock function with given fields: ctx, key -func (_m *Tree) RestoreRecycleItemFunc(ctx context.Context, key string) (*node.Node, func() error, error) { - ret := _m.Called(ctx, key) - - var r0 *node.Node - if rf, ok := ret.Get(0).(func(context.Context, string) *node.Node); ok { - r0 = rf(ctx, key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*node.Node) - } - } - - var r1 func() error - if rf, ok := ret.Get(1).(func(context.Context, string) func() error); ok { - r1 = rf(ctx, key) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(func() error) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context, string) error); ok { - r2 = rf(ctx, key) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// Setup provides a mock function with given fields: owner -func (_m *Tree) Setup(owner string) error { - ret := _m.Called(owner) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(owner) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// WriteBlob provides a mock function with given fields: key, reader -func (_m *Tree) WriteBlob(key string, reader io.Reader) error { - ret := _m.Called(key, reader) - - var r0 error - if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(key, reader) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go deleted file mode 100644 index bc1b1317e2..0000000000 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ /dev/null @@ -1,995 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "crypto/md5" - "encoding/hex" - "fmt" - "hash" - "io" - "io/fs" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/internal/grpc/services/storageprovider" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/mime" - "github.com/cs3org/reva/pkg/storage/utils/ace" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/utils" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// Define keys and values used in the node metadata. -const ( - FavoriteKey = "http://owncloud.org/ns/favorite" - ShareTypesKey = "http://owncloud.org/ns/share-types" - ChecksumsKey = "http://owncloud.org/ns/checksums" - UserShareType = "0" - QuotaKey = "quota" - - QuotaUncalculated = "-1" - QuotaUnknown = "-2" - QuotaUnlimited = "-3" -) - -// Node represents a node in the tree and provides methods to get a Parent or Child instance. -type Node struct { - ParentID string - ID string - Name string - Blobsize int64 - BlobID string - owner *userpb.UserId - Exists bool - SpaceRoot *Node - - lu PathLookup -} - -// PathLookup defines the interface for the lookup component. -type PathLookup interface { - RootNode(ctx context.Context) (node *Node, err error) - HomeOrRootNode(ctx context.Context) (node *Node, err error) - - InternalRoot() string - InternalPath(ID string) string - Path(ctx context.Context, n *Node) (path string, err error) - ShareFolder() string -} - -// New returns a new instance of Node. -func New(id, parentID, name string, blobsize int64, blobID string, owner *userpb.UserId, lu PathLookup) *Node { - if blobID == "" { - blobID = uuid.New().String() - } - return &Node{ - ID: id, - ParentID: parentID, - Name: name, - Blobsize: blobsize, - owner: owner, - lu: lu, - BlobID: blobID, - } -} - -// ChangeOwner sets the owner of n to newOwner. -func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { - nodePath := n.InternalPath() - n.owner = new - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte(new.OpaqueId)); err != nil { - return errors.Wrap(err, "decomposedfs: could not reset owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte(new.Idp)); err != nil { - return errors.Wrap(err, "decomposedfs: could not reset owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte(utils.UserTypeToString(new.Type))); err != nil { - return errors.Wrap(err, "decomposedfs: could not reset owner idp attribute") - } - - return -} - -// SetMetadata populates a given key with its value. -// Note that consumers should be aware of the metadata options on xattrs.go. -func (n *Node) SetMetadata(key string, val string) (err error) { - nodePath := n.InternalPath() - if err := xattr.Set(nodePath, key, []byte(val)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set parentid attribute") - } - return nil -} - -// WriteMetadata writes the Node metadata to disk. -func (n *Node) WriteMetadata(owner *userpb.UserId) (err error) { - nodePath := n.InternalPath() - if err = xattr.Set(nodePath, xattrs.ParentidAttr, []byte(n.ParentID)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set parentid attribute") - } - if err = xattr.Set(nodePath, xattrs.NameAttr, []byte(n.Name)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set name attribute") - } - if err = xattr.Set(nodePath, xattrs.BlobIDAttr, []byte(n.BlobID)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set blobid attribute") - } - if err = xattr.Set(nodePath, xattrs.BlobsizeAttr, []byte(fmt.Sprintf("%d", n.Blobsize))); err != nil { - return errors.Wrap(err, "decomposedfs: could not set blobsize attribute") - } - if owner == nil { - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte("")); err != nil { - return errors.Wrap(err, "decomposedfs: could not set empty owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte("")); err != nil { - return errors.Wrap(err, "decomposedfs: could not set empty owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte("")); err != nil { - return errors.Wrap(err, "decomposedfs: could not set empty owner type attribute") - } - } else { - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte(owner.OpaqueId)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte(owner.Idp)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte(utils.UserTypeToString(owner.Type))); err != nil { - return errors.Wrap(err, "decomposedfs: could not set owner idp attribute") - } - } - return -} - -// ReadNode creates a new instance from an id and checks if it exists. -func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error) { - n = &Node{ - lu: lu, - ID: id, - } - - nodePath := n.InternalPath() - - // lookup parent id in extended attributes - var attrBytes []byte - attrBytes, err = xattr.Get(nodePath, xattrs.ParentidAttr) - switch { - case err == nil: - n.ParentID = string(attrBytes) - case isAttrUnset(err): - return nil, errtypes.InternalError(err.Error()) - case isNotFound(err): - return n, nil // swallow not found, the node defaults to exists = false - default: - return nil, errtypes.InternalError(err.Error()) - } - - // check if this is a space root - if _, err = xattr.Get(nodePath, xattrs.SpaceNameAttr); err == nil { - n.SpaceRoot = n - } - // lookup name in extended attributes - if attrBytes, err = xattr.Get(nodePath, xattrs.NameAttr); err == nil { - n.Name = string(attrBytes) - } else { - return - } - // lookup blobID in extended attributes - if attrBytes, err = xattr.Get(nodePath, xattrs.BlobIDAttr); err == nil { - n.BlobID = string(attrBytes) - } else { - return - } - // Lookup blobsize - var blobSize int64 - if blobSize, err = ReadBlobSizeAttr(nodePath); err == nil { - n.Blobsize = blobSize - } else { - return - } - - // Check if parent exists. Otherwise this node is part of a deleted subtree - _, err = os.Stat(lu.InternalPath(n.ParentID)) - if err != nil { - if isNotFound(err) { - return nil, errtypes.NotFound(err.Error()) - } - return nil, err - } - n.Exists = true - return -} - -// The os error is buried inside the fs.PathError error. -func isNotDir(err error) bool { - if perr, ok := err.(*fs.PathError); ok { - if serr, ok2 := perr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOTDIR - } - } - return false -} - -// Child returns the child node with the given name. -func (n *Node) Child(ctx context.Context, name string) (*Node, error) { - link, err := os.Readlink(filepath.Join(n.InternalPath(), filepath.Join("/", name))) - if err != nil { - if os.IsNotExist(err) || isNotDir(err) { - c := &Node{ - lu: n.lu, - ParentID: n.ID, - Name: name, - SpaceRoot: n.SpaceRoot, - } - return c, nil // if the file does not exist we return a node that has Exists = false - } - - return nil, errors.Wrap(err, "decomposedfs: Wrap: readlink error") - } - - var c *Node - if strings.HasPrefix(link, "../") { - c, err = ReadNode(ctx, n.lu, filepath.Base(link)) - if err != nil { - return nil, errors.Wrap(err, "could not read child node") - } - c.SpaceRoot = n.SpaceRoot - } else { - return nil, fmt.Errorf("decomposedfs: expected '../ prefix, got' %+v", link) - } - - return c, nil -} - -// Parent returns the parent node. -func (n *Node) Parent() (p *Node, err error) { - if n.ParentID == "" { - return nil, fmt.Errorf("decomposedfs: root has no parent") - } - p = &Node{ - lu: n.lu, - ID: n.ParentID, - SpaceRoot: n.SpaceRoot, - } - - parentPath := n.lu.InternalPath(n.ParentID) - - // lookup parent id in extended attributes - var attrBytes []byte - if attrBytes, err = xattr.Get(parentPath, xattrs.ParentidAttr); err == nil { - p.ParentID = string(attrBytes) - } else { - return - } - // lookup name in extended attributes - if attrBytes, err = xattr.Get(parentPath, xattrs.NameAttr); err == nil { - p.Name = string(attrBytes) - } else { - return - } - - // check node exists - if _, err := os.Stat(parentPath); err == nil { - p.Exists = true - } - return -} - -// Owner returns the cached owner id or reads it from the extended attributes -// TODO can be private as only the AsResourceInfo uses it. -func (n *Node) Owner() (*userpb.UserId, error) { - if n.owner != nil { - return n.owner, nil - } - - owner := &userpb.UserId{} - - // FIXME ... do we return the owner of the reference or the owner of the target? - // we don't really know the owner of the target ... and as the reference may point anywhere we cannot really find out - // but what are the permissions? all? none? the gateway has to fill in? - // TODO what if this is a reference? - nodePath := n.InternalPath() - // lookup parent id in extended attributes - var attrBytes []byte - var err error - // lookup ID in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerIDAttr) - switch { - case err == nil: - owner.OpaqueId = string(attrBytes) - case isAttrUnset(err), isNotFound(err): - fallthrough - default: - return nil, err - } - - // lookup IDP in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerIDPAttr) - switch { - case err == nil: - owner.Idp = string(attrBytes) - case isAttrUnset(err), isNotFound(err): - fallthrough - default: - return nil, err - } - - // lookup type in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerTypeAttr) - switch { - case err == nil: - owner.Type = utils.UserTypeMap(string(attrBytes)) - case isAttrUnset(err), isNotFound(err): - fallthrough - default: - // TODO the user type defaults to invalid, which is the case - err = nil - } - - n.owner = owner - return n.owner, err -} - -// PermissionSet returns the permission set for the current user -// the parent nodes are not taken into account. -func (n *Node) PermissionSet(ctx context.Context) provider.ResourcePermissions { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("no user in context, returning default permissions") - return NoPermissions() - } - if o, _ := n.Owner(); utils.UserEqual(u.Id, o) { - return OwnerPermissions() - } - // read the permissions for the current user from the acls of the current node - if np, err := n.ReadUserPermissions(ctx, u); err == nil { - return np - } - return NoPermissions() -} - -// InternalPath returns the internal path of the Node. -func (n *Node) InternalPath() string { - return n.lu.InternalPath(n.ID) -} - -// CalculateEtag returns a hash of fileid + tmtime (or mtime). -func CalculateEtag(nodeID string, tmTime time.Time) (string, error) { - return calculateEtag(nodeID, tmTime) -} - -// calculateEtag returns a hash of fileid + tmtime (or mtime). -func calculateEtag(nodeID string, tmTime time.Time) (string, error) { - h := md5.New() - if _, err := io.WriteString(h, nodeID); err != nil { - return "", err - } - if tb, err := tmTime.UTC().MarshalBinary(); err == nil { - if _, err := h.Write(tb); err != nil { - return "", err - } - } else { - return "", err - } - return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil -} - -// SetMtime sets the mtime and atime of a node. -func (n *Node) SetMtime(ctx context.Context, mtime string) error { - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - if mt, err := parseMTime(mtime); err == nil { - nodePath := n.lu.InternalPath(n.ID) - // updating mtime also updates atime - if err := os.Chtimes(nodePath, mt, mt); err != nil { - sublog.Error().Err(err). - Time("mtime", mt). - Msg("could not set mtime") - return errors.Wrap(err, "could not set mtime") - } - } else { - sublog.Error().Err(err). - Str("mtime", mtime). - Msg("could not parse mtime") - return errors.Wrap(err, "could not parse mtime") - } - return nil -} - -// SetEtag sets the temporary etag of a node if it differs from the current etag. -func (n *Node) SetEtag(ctx context.Context, val string) (err error) { - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - nodePath := n.lu.InternalPath(n.ID) - var tmTime time.Time - if tmTime, err = n.GetTMTime(); err != nil { - // no tmtime, use mtime - var fi os.FileInfo - if fi, err = os.Lstat(nodePath); err != nil { - return - } - tmTime = fi.ModTime() - } - var etag string - if etag, err = calculateEtag(n.ID, tmTime); err != nil { - return - } - - // sanitize etag - val = fmt.Sprintf("\"%s\"", strings.Trim(val, "\"")) - if etag == val { - sublog.Debug(). - Str("etag", val). - Msg("ignoring request to update identical etag") - return nil - } - // etag is only valid until the calculated etag changes, is part of propagation - return xattr.Set(nodePath, xattrs.TmpEtagAttr, []byte(val)) -} - -// SetFavorite sets the favorite for the current user -// TODO we should not mess with the user here ... the favorites is now a user specific property for a file -// that cannot be mapped to extended attributes without leaking who has marked a file as a favorite -// it is a specific case of a tag, which is user individual as well -// TODO there are different types of tags -// 1. public that are managed by everyone -// 2. private tags that are only visible to the user -// 3. system tags that are only visible to the system -// 4. group tags that are only visible to a group ... -// urgh ... well this can be solved using different namespaces -// 1. public = p: -// 2. private = u:: for user specific -// 3. system = s: for system -// 4. group = g:: -// 5. app? = a:: for apps? -// obviously this only is secure when the u/s/g/a namespaces are not accessible by users in the filesystem -// public tags can be mapped to extended attributes. -func (n *Node) SetFavorite(uid *userpb.UserId, val string) error { - nodePath := n.lu.InternalPath(n.ID) - // the favorite flag is specific to the user, so we need to incorporate the userid - fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - return xattr.Set(nodePath, fa, []byte(val)) -} - -// AsResourceInfo return the node as CS3 ResourceInfo. -func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissions, mdKeys []string, returnBasename bool) (ri *provider.ResourceInfo, err error) { - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - - var fn string - nodePath := n.lu.InternalPath(n.ID) - - var fi os.FileInfo - - nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID - if fi, err = os.Lstat(nodePath); err != nil { - return - } - - var target []byte - switch { - case fi.IsDir(): - if target, err = xattr.Get(nodePath, xattrs.ReferenceAttr); err == nil { - nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } else { - nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - case fi.Mode().IsRegular(): - nodeType = provider.ResourceType_RESOURCE_TYPE_FILE - case fi.Mode()&os.ModeSymlink != 0: - nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK - // TODO reference using ext attr on a symlink - // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } - - id := &provider.ResourceId{OpaqueId: n.ID} - - if returnBasename { - fn = n.Name - } else { - fn, err = n.lu.Path(ctx, n) - if err != nil { - return nil, err - } - } - - ri = &provider.ResourceInfo{ - Id: id, - Path: fn, - Type: nodeType, - MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), - Size: uint64(n.Blobsize), - Target: string(target), - PermissionSet: rp, - } - - if nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - ts, err := n.GetTreeSize() - if err == nil { - ri.Size = ts - } else { - ri.Size = 0 // make dirs always return 0 if it is unknown - sublog.Debug().Err(err).Msg("could not read treesize") - } - } - - if ri.Owner, err = n.Owner(); err != nil { - sublog.Debug().Err(err).Msg("could not determine owner") - } - - // TODO make etag of files use fileid and checksum - - var tmTime time.Time - if tmTime, err = n.GetTMTime(); err != nil { - // no tmtime, use mtime - tmTime = fi.ModTime() - } - - // use temporary etag if it is set - if b, err := xattr.Get(nodePath, xattrs.TmpEtagAttr); err == nil { - ri.Etag = fmt.Sprintf(`"%x"`, string(b)) // TODO why do we convert string(b)? is the temporary etag stored as string? -> should we use bytes? use hex.EncodeToString? - } else if ri.Etag, err = calculateEtag(n.ID, tmTime); err != nil { - sublog.Debug().Err(err).Msg("could not calculate etag") - } - - // mtime uses tmtime if present - // TODO expose mtime and tmtime separately? - un := tmTime.UnixNano() - ri.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - - mdKeysMap := make(map[string]struct{}) - for _, k := range mdKeys { - mdKeysMap[k] = struct{}{} - } - - var returnAllKeys bool - if _, ok := mdKeysMap["*"]; len(mdKeys) == 0 || ok { - returnAllKeys = true - } - - metadata := map[string]string{} - - // read favorite flag for the current user - if _, ok := mdKeysMap[FavoriteKey]; returnAllKeys || ok { - favorite := "" - if u, ok := ctxpkg.ContextGetUser(ctx); ok { - // the favorite flag is specific to the user, so we need to incorporate the userid - if uid := u.GetId(); uid != nil { - fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if val, err := xattr.Get(nodePath, fa); err == nil { - sublog.Debug(). - Str("favorite", fa). - Msg("found favorite flag") - favorite = string(val) - } - } else { - sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id") - } - } else { - sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("error getting user from ctx") - } - metadata[FavoriteKey] = favorite - } - - // share indicator - if _, ok := mdKeysMap[ShareTypesKey]; returnAllKeys || ok { - if n.hasUserShares(ctx) { - metadata[ShareTypesKey] = UserShareType - } - } - - // checksums - if _, ok := mdKeysMap[ChecksumsKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_FILE) && returnAllKeys || ok { - // TODO which checksum was requested? sha1 adler32 or md5? for now hardcode sha1? - readChecksumIntoResourceChecksum(ctx, nodePath, storageprovider.XSSHA1, ri) - readChecksumIntoOpaque(ctx, nodePath, storageprovider.XSMD5, ri) - readChecksumIntoOpaque(ctx, nodePath, storageprovider.XSAdler32, ri) - } - // quota - if _, ok := mdKeysMap[QuotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok { - var quotaPath string - if n.SpaceRoot == nil { - root, err := n.lu.HomeOrRootNode(ctx) - if err == nil { - quotaPath = root.InternalPath() - } else { - sublog.Debug().Err(err).Msg("error determining the space root node for quota") - } - } else { - quotaPath = n.SpaceRoot.InternalPath() - } - if quotaPath != "" { - readQuotaIntoOpaque(ctx, quotaPath, ri) - } - } - - // only read the requested metadata attributes - attrs, err := xattr.List(nodePath) - if err != nil { - sublog.Error().Err(err).Msg("error getting list of extended attributes") - } else { - for i := range attrs { - // filter out non-custom properties - if !strings.HasPrefix(attrs[i], xattrs.MetadataPrefix) { - continue - } - // only read when key was requested - k := attrs[i][len(xattrs.MetadataPrefix):] - if _, ok := mdKeysMap[k]; returnAllKeys || ok { - if val, err := xattr.Get(nodePath, attrs[i]); err == nil { - metadata[k] = string(val) - } else { - sublog.Error().Err(err). - Str("entry", attrs[i]). - Msg("error retrieving xattr metadata") - } - } - } - } - ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ - Metadata: metadata, - } - - sublog.Debug(). - Interface("ri", ri). - Msg("AsResourceInfo") - - return ri, nil -} - -func readChecksumIntoResourceChecksum(ctx context.Context, nodePath, algo string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.ChecksumPrefix+algo) - switch { - case err == nil: - ri.Checksum = &provider.ResourceChecksum{ - Type: storageprovider.PKG2GRPCXS(algo), - Sum: hex.EncodeToString(v), - } - case isAttrUnset(err): - appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("checksum not set") - case isNotFound(err): - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("file not fount") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("could not read checksum") - } -} - -func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.ChecksumPrefix+algo) - switch { - case err == nil: - if ri.Opaque == nil { - ri.Opaque = &types.Opaque{ - Map: map[string]*types.OpaqueEntry{}, - } - } - ri.Opaque.Map[algo] = &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte(hex.EncodeToString(v)), - } - case isAttrUnset(err): - appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("checksum not set") - case isNotFound(err): - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("file not fount") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("could not read checksum") - } -} - -// quota is always stored on the root node. -func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.QuotaAttr) - switch { - case err == nil: - // make sure we have a proper signed int - // we use the same magic numbers to indicate: - // -1 = uncalculated - // -2 = unknown - // -3 = unlimited - if _, err := strconv.ParseInt(string(v), 10, 64); err == nil { - if ri.Opaque == nil { - ri.Opaque = &types.Opaque{ - Map: map[string]*types.OpaqueEntry{}, - } - } - ri.Opaque.Map[QuotaKey] = &types.OpaqueEntry{ - Decoder: "plain", - Value: v, - } - } else { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("quota", string(v)).Msg("malformed quota") - } - case isAttrUnset(err): - appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("quota not set") - case isNotFound(err): - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("file not found when reading quota") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read quota") - } -} - -// HasPropagation checks if the propagation attribute exists and is set to "1". -func (n *Node) HasPropagation() (propagation bool) { - if b, err := xattr.Get(n.lu.InternalPath(n.ID), xattrs.PropagationAttr); err == nil { - return string(b) == "1" - } - return false -} - -// GetTMTime reads the tmtime from the extended attributes. -func (n *Node) GetTMTime() (tmTime time.Time, err error) { - var b []byte - if b, err = xattr.Get(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr); err != nil { - return - } - return time.Parse(time.RFC3339Nano, string(b)) -} - -// SetTMTime writes the tmtime to the extended attributes. -func (n *Node) SetTMTime(t time.Time) (err error) { - return xattr.Set(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano))) -} - -// GetTreeSize reads the treesize from the extended attributes. -func (n *Node) GetTreeSize() (treesize uint64, err error) { - var b []byte - if b, err = xattr.Get(n.InternalPath(), xattrs.TreesizeAttr); err != nil { - return - } - return strconv.ParseUint(string(b), 10, 64) -} - -// SetTreeSize writes the treesize to the extended attributes. -func (n *Node) SetTreeSize(ts uint64) (err error) { - return xattr.Set(n.InternalPath(), xattrs.TreesizeAttr, []byte(strconv.FormatUint(ts, 10))) -} - -// SetChecksum writes the checksum with the given checksum type to the extended attributes. -func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { - return xattr.Set(n.lu.InternalPath(n.ID), xattrs.ChecksumPrefix+csType, h.Sum(nil)) -} - -// UnsetTempEtag removes the temporary etag attribute. -func (n *Node) UnsetTempEtag() (err error) { - if err = xattr.Remove(n.lu.InternalPath(n.ID), xattrs.TmpEtagAttr); err != nil { - if e, ok := err.(*xattr.Error); ok && (e.Err.Error() == "no data available" || - // darwin - e.Err.Error() == "attribute not found") { - return nil - } - } - return err -} - -// ReadUserPermissions will assemble the permissions for the current user on the given node without parent nodes. -func (n *Node) ReadUserPermissions(ctx context.Context, u *userpb.User) (ap provider.ResourcePermissions, err error) { - // check if the current user is the owner - o, err := n.Owner() - if err != nil { - // TODO check if a parent folder has the owner set? - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not determine owner, returning default permissions") - return NoPermissions(), err - } - if o.OpaqueId == "" { - // this happens for root nodes in the storage. the extended attributes are set to emptystring to indicate: no owner - // TODO what if no owner is set but grants are present? - return NoOwnerPermissions(), nil - } - if utils.UserEqual(u.Id, o) { - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("user is owner, returning owner permissions") - return OwnerPermissions(), nil - } - - ap = provider.ResourcePermissions{} - - // for an efficient group lookup convert the list of groups to a map - // groups are just strings ... groupnames ... or group ids ??? AAARGH !!! - groupsMap := make(map[string]bool, len(u.Groups)) - for i := range u.Groups { - groupsMap[u.Groups[i]] = true - } - - var g *provider.Grant - - // we read all grantees from the node - var grantees []string - if grantees, err = n.ListGrantees(ctx); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("error listing grantees") - return NoPermissions(), err - } - - // instead of making n getxattr syscalls we are going to list the acls and filter them here - // we have two options here: - // 1. we can start iterating over the acls / grants on the node or - // 2. we can iterate over the number of groups - // The current implementation tries to be defensive for cases where users have hundreds or thousands of groups, so we iterate over the existing acls. - userace := xattrs.GrantPrefix + xattrs.UserAcePrefix + u.Id.OpaqueId - userFound := false - for i := range grantees { - switch { - // we only need to find the user once - case !userFound && grantees[i] == userace: - g, err = n.ReadGrant(ctx, grantees[i]) - case strings.HasPrefix(grantees[i], xattrs.GrantPrefix+xattrs.GroupAcePrefix): // only check group grantees - gr := strings.TrimPrefix(grantees[i], xattrs.GrantPrefix+xattrs.GroupAcePrefix) - if groupsMap[gr] { - g, err = n.ReadGrant(ctx, grantees[i]) - } else { - // no need to check attribute - continue - } - default: - // no need to check attribute - continue - } - - switch { - case err == nil: - AddPermissions(&ap, g.GetPermissions()) - case isAttrUnset(err): - err = nil - appctx.GetLogger(ctx).Error().Interface("node", n).Str("grant", grantees[i]).Interface("grantees", grantees).Msg("grant vanished from node after listing") - // continue with next segment - default: - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Str("grant", grantees[i]).Msg("error reading permissions") - // continue with next segment - } - } - - appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Interface("node", n).Interface("user", u).Msg("returning aggregated permissions") - return ap, nil -} - -// ListGrantees lists the grantees of the current node -// We don't want to wast time and memory by creating grantee objects. -// The function will return a list of opaque strings that can be used to make a ReadGrant call. -func (n *Node) ListGrantees(ctx context.Context) (grantees []string, err error) { - var attrs []string - if attrs, err = xattr.List(n.InternalPath()); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("error listing attributes") - return nil, err - } - for i := range attrs { - if strings.HasPrefix(attrs[i], xattrs.GrantPrefix) { - grantees = append(grantees, attrs[i]) - } - } - return -} - -// ReadGrant reads a CS3 grant. -func (n *Node) ReadGrant(ctx context.Context, grantee string) (g *provider.Grant, err error) { - var b []byte - if b, err = xattr.Get(n.InternalPath(), grantee); err != nil { - return nil, err - } - var e *ace.ACE - if e, err = ace.Unmarshal(strings.TrimPrefix(grantee, xattrs.GrantPrefix), b); err != nil { - return nil, err - } - return e.Grant(), nil -} - -// ReadBlobSizeAttr reads the blobsize from the xattrs. -func ReadBlobSizeAttr(path string) (int64, error) { - attrBytes, err := xattr.Get(path, xattrs.BlobsizeAttr) - if err != nil { - return 0, errors.Wrapf(err, "error reading blobsize xattr") - } - blobSize, err := strconv.ParseInt(string(attrBytes), 10, 64) - if err != nil { - return 0, errors.Wrapf(err, "invalid blobsize xattr format") - } - return blobSize, nil -} - -func (n *Node) hasUserShares(ctx context.Context) bool { - g, err := n.ListGrantees(ctx) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("hasUserShares: listGrantees") - return false - } - - for i := range g { - if strings.Contains(g[i], xattrs.GrantPrefix+xattrs.UserAcePrefix) { - return true - } - } - return false -} - -func parseMTime(v string) (t time.Time, err error) { - p := strings.SplitN(v, ".", 2) - var sec, nsec int64 - if sec, err = strconv.ParseInt(p[0], 10, 64); err == nil { - if len(p) > 1 { - nsec, err = strconv.ParseInt(p[1], 10, 64) - } - } - return time.Unix(sec, nsec), err -} - -// FindStorageSpaceRoot calls n.Parent() and climbs the tree -// until it finds the space root node and adds it to the node. -func (n *Node) FindStorageSpaceRoot() error { - var err error - // remember the node we ask for and use parent to climb the tree - parent := n - for parent.ParentID != "" { - if parent, err = parent.Parent(); err != nil { - return err - } - if IsSpaceRoot(parent) { - n.SpaceRoot = parent - break - } - } - return nil -} - -// IsSpaceRoot checks if the node is a space root. -func IsSpaceRoot(r *Node) bool { - path := r.InternalPath() - if spaceNameBytes, err := xattr.Get(path, xattrs.SpaceNameAttr); err == nil { - if string(spaceNameBytes) != "" { - return true - } - } - return false -} - -// CheckQuota checks if both disk space and available quota are sufficient. -var CheckQuota = func(spaceRoot *Node, fileSize uint64) (quotaSufficient bool, err error) { - used, _ := spaceRoot.GetTreeSize() - if !enoughDiskSpace(spaceRoot.InternalPath(), fileSize) { - return false, errtypes.InsufficientStorage("disk full") - } - quotaByte, _ := xattr.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) - var total uint64 - if quotaByte == nil { - // if quota is not set, it means unlimited - return true, nil - } - total, _ = strconv.ParseUint(string(quotaByte), 10, 64) - // if total is smaller than used, total-used could overflow and be bigger than fileSize - if fileSize > total-used || total < used { - return false, errtypes.InsufficientStorage("quota exceeded") - } - return true, nil -} - -func enoughDiskSpace(path string, fileSize uint64) bool { - avalB, err := GetAvailableSize(path) - if err != nil { - return false - } - return avalB > fileSize -} diff --git a/pkg/storage/utils/decomposedfs/node/node_suite_test.go b/pkg/storage/utils/decomposedfs/node/node_suite_test.go deleted file mode 100644 index 0030f54278..0000000000 --- a/pkg/storage/utils/decomposedfs/node/node_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestNode(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Node Suite") -} diff --git a/pkg/storage/utils/decomposedfs/node/node_test.go b/pkg/storage/utils/decomposedfs/node/node_test.go deleted file mode 100644 index 3ece643caa..0000000000 --- a/pkg/storage/utils/decomposedfs/node/node_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node_test - -import ( - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Node", func() { - var ( - env *helpers.TestEnv - - id string - name string - ) - - BeforeEach(func() { - var err error - env, err = helpers.NewTestEnv() - Expect(err).ToNot(HaveOccurred()) - - id = "fooId" - name = "foo" - }) - - AfterEach(func() { - if env != nil { - env.Cleanup() - } - }) - - Describe("New", func() { - It("generates unique blob ids if none are given", func() { - n1 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup) - n2 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup) - - Expect(len(n1.BlobID)).To(Equal(36)) - Expect(n1.BlobID).ToNot(Equal(n2.BlobID)) - }) - }) - - Describe("ReadNode", func() { - It("reads the blobID from the xattrs", func() { - lookupNode, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - - n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.ID) - Expect(err).ToNot(HaveOccurred()) - Expect(n.BlobID).To(Equal("file1-blobid")) - }) - }) - - Describe("WriteMetadata", func() { - It("writes all xattrs", func() { - n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - - blobsize := 239485734 - n.Name = "TestName" - n.BlobID = "TestBlobID" - n.Blobsize = int64(blobsize) - owner := &userpb.UserId{ - Idp: "testidp", - OpaqueId: "testuserid", - Type: userpb.UserType_USER_TYPE_PRIMARY, - } - - err = n.WriteMetadata(owner) - Expect(err).ToNot(HaveOccurred()) - n2, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - Expect(n2.Name).To(Equal("TestName")) - Expect(n2.BlobID).To(Equal("TestBlobID")) - Expect(n2.Blobsize).To(Equal(int64(blobsize))) - }) - }) - - Describe("Parent", func() { - It("returns the parent node", func() { - child, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/subdir1", false) - Expect(err).ToNot(HaveOccurred()) - Expect(child).ToNot(BeNil()) - - parent, err := child.Parent() - Expect(err).ToNot(HaveOccurred()) - Expect(parent).ToNot(BeNil()) - Expect(parent.ID).To(Equal(child.ParentID)) - }) - }) - - Describe("Child", func() { - var ( - parent *node.Node - ) - - BeforeEach(func() { - var err error - parent, err = env.Lookup.NodeFromPath(env.Ctx, "/dir1", false) - Expect(err).ToNot(HaveOccurred()) - Expect(parent).ToNot(BeNil()) - }) - - It("returns an empty node if the child does not exist", func() { - child, err := parent.Child(env.Ctx, "does-not-exist") - Expect(err).ToNot(HaveOccurred()) - Expect(child).ToNot(BeNil()) - Expect(child.Exists).To(BeFalse()) - }) - - It("returns a directory node with all metadata", func() { - child, err := parent.Child(env.Ctx, "subdir1") - Expect(err).ToNot(HaveOccurred()) - Expect(child).ToNot(BeNil()) - Expect(child.Exists).To(BeTrue()) - Expect(child.ParentID).To(Equal(parent.ID)) - Expect(child.Name).To(Equal("subdir1")) - Expect(child.Blobsize).To(Equal(int64(0))) - }) - - It("returns a file node with all metadata", func() { - child, err := parent.Child(env.Ctx, "file1") - Expect(err).ToNot(HaveOccurred()) - Expect(child).ToNot(BeNil()) - Expect(child.Exists).To(BeTrue()) - Expect(child.ParentID).To(Equal(parent.ID)) - Expect(child.Name).To(Equal("file1")) - Expect(child.Blobsize).To(Equal(int64(1234))) - }) - - It("handles (broken) links including file segments by returning an non-existent node", func() { - child, err := parent.Child(env.Ctx, "file1/broken") - Expect(err).ToNot(HaveOccurred()) - Expect(child).ToNot(BeNil()) - Expect(child.Exists).To(BeFalse()) - }) - }) - - Describe("AsResourceInfo", func() { - var ( - n *node.Node - ) - - BeforeEach(func() { - var err error - n, err = env.Lookup.NodeFromPath(env.Ctx, "dir1/file1", false) - Expect(err).ToNot(HaveOccurred()) - }) - - Describe("the Etag field", func() { - It("is set", func() { - perms := node.OwnerPermissions() - ri, err := n.AsResourceInfo(env.Ctx, &perms, []string{}, false) - Expect(err).ToNot(HaveOccurred()) - Expect(len(ri.Etag)).To(Equal(34)) - }) - - It("changes when the tmtime is set", func() { - perms := node.OwnerPermissions() - ri, err := n.AsResourceInfo(env.Ctx, &perms, []string{}, false) - Expect(err).ToNot(HaveOccurred()) - Expect(len(ri.Etag)).To(Equal(34)) - before := ri.Etag - - Expect(n.SetTMTime(time.Now().UTC())).To(Succeed()) - - ri, err = n.AsResourceInfo(env.Ctx, &perms, []string{}, false) - Expect(err).ToNot(HaveOccurred()) - Expect(len(ri.Etag)).To(Equal(34)) - Expect(ri.Etag).ToNot(Equal(before)) - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/node/node_unix.go b/pkg/storage/utils/decomposedfs/node/node_unix.go deleted file mode 100644 index 90f8f6d63a..0000000000 --- a/pkg/storage/utils/decomposedfs/node/node_unix.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build !windows -// +build !windows - -package node - -import "syscall" - -// GetAvailableSize stats the filesystem and return the available bytes. -func GetAvailableSize(path string) (uint64, error) { - stat := syscall.Statfs_t{} - err := syscall.Statfs(path, &stat) - if err != nil { - return 0, err - } - return stat.Bavail * uint64(stat.Bsize), nil -} diff --git a/pkg/storage/utils/decomposedfs/node/node_windows.go b/pkg/storage/utils/decomposedfs/node/node_windows.go deleted file mode 100644 index db11894b29..0000000000 --- a/pkg/storage/utils/decomposedfs/node/node_windows.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build windows -// +build windows - -package node - -import "golang.org/x/sys/windows" - -// GetAvailableSize stats the filesystem and return the available bytes -func GetAvailableSize(path string) (uint64, error) { - var free, total, avail uint64 - pathPtr, err := windows.UTF16PtrFromString(path) - if err != nil { - return 0, err - } - err = windows.GetDiskFreeSpaceEx(pathPtr, &avail, &total, &free) - if err != nil { - return 0, err - } - return avail, nil -} diff --git a/pkg/storage/utils/decomposedfs/node/permissions.go b/pkg/storage/utils/decomposedfs/node/permissions.go deleted file mode 100644 index 66662c9740..0000000000 --- a/pkg/storage/utils/decomposedfs/node/permissions.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "strings" - "syscall" - - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/utils" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// NoPermissions represents an empty set of permissions. -func NoPermissions() provider.ResourcePermissions { - return provider.ResourcePermissions{} -} - -// NoOwnerPermissions defines permissions for nodes that don't have an owner set, eg the root node. -func NoOwnerPermissions() provider.ResourcePermissions { - return provider.ResourcePermissions{ - Stat: true, - } -} - -// ShareFolderPermissions defines permissions for the shared jail. -func ShareFolderPermissions() provider.ResourcePermissions { - return provider.ResourcePermissions{ - // read permissions - ListContainer: true, - Stat: true, - InitiateFileDownload: true, - GetPath: true, - GetQuota: true, - ListFileVersions: true, - } -} - -// OwnerPermissions defines permissions for nodes owned by the user. -func OwnerPermissions() provider.ResourcePermissions { - return provider.ResourcePermissions{ - // all permissions - AddGrant: true, - CreateContainer: true, - Delete: true, - GetPath: true, - GetQuota: true, - InitiateFileDownload: true, - InitiateFileUpload: true, - ListContainer: true, - ListFileVersions: true, - ListGrants: true, - ListRecycle: true, - Move: true, - PurgeRecycle: true, - RemoveGrant: true, - RestoreFileVersion: true, - RestoreRecycleItem: true, - Stat: true, - UpdateGrant: true, - } -} - -// Permissions implements permission checks. -type Permissions struct { - lu PathLookup -} - -// NewPermissions returns a new Permissions instance. -func NewPermissions(lu PathLookup) *Permissions { - return &Permissions{ - lu: lu, - } -} - -// AssemblePermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes. -func (p *Permissions) AssemblePermissions(ctx context.Context, n *Node) (ap provider.ResourcePermissions, err error) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("no user in context, returning default permissions") - return NoPermissions(), nil - } - // check if the current user is the owner - o, err := n.Owner() - if err != nil { - // TODO check if a parent folder has the owner set? - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not determine owner, returning default permissions") - return NoPermissions(), err - } - if o.OpaqueId == "" { - // this happens for root nodes in the storage. the extended attributes are set to emptystring to indicate: no owner - // TODO what if no owner is set but grants are present? - return NoOwnerPermissions(), nil - } - if utils.UserEqual(u.Id, o) { - lp, err := n.lu.Path(ctx, n) - if err == nil && lp == n.lu.ShareFolder() { - return ShareFolderPermissions(), nil - } - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("user is owner, returning owner permissions") - return OwnerPermissions(), nil - } - // determine root - var rn *Node - if rn, err = p.lu.RootNode(ctx); err != nil { - return NoPermissions(), err - } - - cn := n - - ap = provider.ResourcePermissions{} - - // for an efficient group lookup convert the list of groups to a map - // groups are just strings ... groupnames ... or group ids ??? AAARGH !!! - groupsMap := make(map[string]bool, len(u.Groups)) - for i := range u.Groups { - groupsMap[u.Groups[i]] = true - } - - // for all segments, starting at the leaf - for cn.ID != rn.ID { - if np, err := cn.ReadUserPermissions(ctx, u); err == nil { - AddPermissions(&ap, &np) - } else { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", cn).Msg("error reading permissions") - // continue with next segment - } - if cn, err = cn.Parent(); err != nil { - return ap, errors.Wrap(err, "decomposedfs: error getting parent "+cn.ParentID) - } - } - - appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Interface("node", n).Interface("user", u).Msg("returning agregated permissions") - return ap, nil -} - -// AddPermissions merges a set of permissions into another -// TODO we should use a bitfield for this ... -func AddPermissions(l *provider.ResourcePermissions, r *provider.ResourcePermissions) { - l.AddGrant = l.AddGrant || r.AddGrant - l.CreateContainer = l.CreateContainer || r.CreateContainer - l.Delete = l.Delete || r.Delete - l.GetPath = l.GetPath || r.GetPath - l.GetQuota = l.GetQuota || r.GetQuota - l.InitiateFileDownload = l.InitiateFileDownload || r.InitiateFileDownload - l.InitiateFileUpload = l.InitiateFileUpload || r.InitiateFileUpload - l.ListContainer = l.ListContainer || r.ListContainer - l.ListFileVersions = l.ListFileVersions || r.ListFileVersions - l.ListGrants = l.ListGrants || r.ListGrants - l.ListRecycle = l.ListRecycle || r.ListRecycle - l.Move = l.Move || r.Move - l.PurgeRecycle = l.PurgeRecycle || r.PurgeRecycle - l.RemoveGrant = l.RemoveGrant || r.RemoveGrant - l.RestoreFileVersion = l.RestoreFileVersion || r.RestoreFileVersion - l.RestoreRecycleItem = l.RestoreRecycleItem || r.RestoreRecycleItem - l.Stat = l.Stat || r.Stat - l.UpdateGrant = l.UpdateGrant || r.UpdateGrant -} - -// HasPermission call check() for every node up to the root until check returns true. -func (p *Permissions) HasPermission(ctx context.Context, n *Node, check func(*provider.ResourcePermissions) bool) (can bool, err error) { - var u *userv1beta1.User - var perms *provider.ResourcePermissions - if u, perms = p.getUserAndPermissions(ctx, n); perms != nil { - return check(perms), nil - } - - // determine root - var rn *Node - if rn, err = p.lu.RootNode(ctx); err != nil { - return false, err - } - - cn := n - - // for an efficient group lookup convert the list of groups to a map - // groups are just strings ... groupnames ... or group ids ??? AAARGH !!! - groupsMap := make(map[string]bool, len(u.Groups)) - for i := range u.Groups { - groupsMap[u.Groups[i]] = true - } - - var g *provider.Grant - // for all segments, starting at the leaf - for cn.ID != rn.ID { - var grantees []string - if grantees, err = cn.ListGrantees(ctx); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", cn).Msg("error listing grantees") - return false, err - } - - userace := xattrs.GrantPrefix + xattrs.UserAcePrefix + u.Id.OpaqueId - userFound := false - for i := range grantees { - // we only need the find the user once per node - switch { - case !userFound && grantees[i] == userace: - g, err = cn.ReadGrant(ctx, grantees[i]) - case strings.HasPrefix(grantees[i], xattrs.GrantPrefix+xattrs.GroupAcePrefix): - gr := strings.TrimPrefix(grantees[i], xattrs.GrantPrefix+xattrs.GroupAcePrefix) - if groupsMap[gr] { - g, err = cn.ReadGrant(ctx, grantees[i]) - } else { - // no need to check attribute - continue - } - default: - // no need to check attribute - continue - } - - switch { - case err == nil: - appctx.GetLogger(ctx).Debug().Interface("node", cn).Str("grant", grantees[i]).Interface("permissions", g.GetPermissions()).Msg("checking permissions") - if check(g.GetPermissions()) { - return true, nil - } - case isAttrUnset(err): - err = nil - appctx.GetLogger(ctx).Error().Interface("node", cn).Str("grant", grantees[i]).Interface("grantees", grantees).Msg("grant vanished from node after listing") - default: - appctx.GetLogger(ctx).Error().Err(err).Interface("node", cn).Str("grant", grantees[i]).Msg("error reading permissions") - return false, err - } - } - - if cn, err = cn.Parent(); err != nil { - return false, errors.Wrap(err, "decomposedfs: error getting parent "+cn.ParentID) - } - } - - appctx.GetLogger(ctx).Debug().Interface("permissions", NoPermissions()).Interface("node", n).Interface("user", u).Msg("no grant found, returning default permissions") - return false, nil -} - -func (p *Permissions) getUserAndPermissions(ctx context.Context, n *Node) (*userv1beta1.User, *provider.ResourcePermissions) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("no user in context, returning default permissions") - perms := NoPermissions() - return nil, &perms - } - // check if the current user is the owner - o, err := n.Owner() - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not determine owner, returning default permissions") - perms := NoPermissions() - return nil, &perms - } - if o.OpaqueId == "" { - // this happens for root nodes in the storage. the extended attributes are set to emptystring to indicate: no owner - // TODO what if no owner is set but grants are present? - perms := NoOwnerPermissions() - return nil, &perms - } - if utils.UserEqual(u.Id, o) { - appctx.GetLogger(ctx).Debug().Interface("node", n).Msg("user is owner, returning owner permissions") - perms := OwnerPermissions() - return u, &perms - } - return u, nil -} - -// The os not exists error is buried inside the xattr error, -// so we cannot just use os.IsNotExists(). -func isNotFound(err error) bool { - if xerr, ok := err.(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOENT - } - } - return false -} diff --git a/pkg/storage/utils/decomposedfs/node/permissions_darwin.go b/pkg/storage/utils/decomposedfs/node/permissions_darwin.go deleted file mode 100644 index f0b03391fb..0000000000 --- a/pkg/storage/utils/decomposedfs/node/permissions_darwin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build darwin -// +build darwin - -package node - -import ( - "syscall" - - "github.com/pkg/xattr" -) - -func isAttrUnset(err error) bool { - if xerr, ok := err.(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOATTR - } - } - return false -} diff --git a/pkg/storage/utils/decomposedfs/node/permissions_unix.go b/pkg/storage/utils/decomposedfs/node/permissions_unix.go deleted file mode 100644 index 4260a3b002..0000000000 --- a/pkg/storage/utils/decomposedfs/node/permissions_unix.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build !darwin -// +build !darwin - -package node - -import ( - "syscall" - - "github.com/pkg/xattr" -) - -func isAttrUnset(err error) bool { - if xerr, ok := err.(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENODATA - } - } - return false -} diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go deleted file mode 100644 index 66e981163e..0000000000 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package options - -import ( - "path/filepath" - "strings" - - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - // ocis fs works on top of a dir of uuid nodes - Root string `mapstructure:"root"` - - // UserLayout describes the relative path from the storage's root node to the users home node. - UserLayout string `mapstructure:"user_layout"` - - // TODO NodeLayout option to save nodes as eg. nodes/1d/d8/1dd84abf-9466-4e14-bb86-02fc4ea3abcf - ShareFolder string `mapstructure:"share_folder"` - - // EnableHome enables the creation of home directories. - EnableHome bool `mapstructure:"enable_home"` - - // propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node - TreeTimeAccounting bool `mapstructure:"treetime_accounting"` - - // propagate size changes as treesize - TreeSizeAccounting bool `mapstructure:"treesize_accounting"` - - // set an owner for the root node - Owner string `mapstructure:"owner"` - OwnerIDP string `mapstructure:"owner_idp"` - OwnerType string `mapstructure:"owner_type"` - - GatewayAddr string `mapstructure:"gateway_addr"` -} - -// New returns a new Options instance for the given configuration. -func New(m map[string]interface{}) (*Options, error) { - o := &Options{} - if err := mapstructure.Decode(m, o); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - - if o.UserLayout == "" { - o.UserLayout = "{{.Id.OpaqueId}}" - } - // ensure user layout has no starting or trailing / - o.UserLayout = strings.Trim(o.UserLayout, "/") - - if o.ShareFolder == "" { - o.ShareFolder = "/Shares" - } - // ensure share folder always starts with slash - o.ShareFolder = filepath.Join("/", o.ShareFolder) - - // c.DataDirectory should never end in / unless it is the root - o.Root = filepath.Clean(o.Root) - - return o, nil -} diff --git a/pkg/storage/utils/decomposedfs/options/options_suite_test.go b/pkg/storage/utils/decomposedfs/options/options_suite_test.go deleted file mode 100644 index 141f49fbdf..0000000000 --- a/pkg/storage/utils/decomposedfs/options/options_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package options_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestOptions(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Options Suite") -} diff --git a/pkg/storage/utils/decomposedfs/options/options_test.go b/pkg/storage/utils/decomposedfs/options/options_test.go deleted file mode 100644 index 3f5faa6257..0000000000 --- a/pkg/storage/utils/decomposedfs/options/options_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package options_test - -import ( - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Options", func() { - var ( - o *options.Options - config map[string]interface{} - ) - - BeforeEach(func() { - config = map[string]interface{}{} - }) - - Describe("New", func() { - JustBeforeEach(func() { - var err error - o, err = options.New(config) - Expect(err).ToNot(HaveOccurred()) - }) - - It("sets defaults", func() { - Expect(len(o.ShareFolder) > 0).To(BeTrue()) - Expect(len(o.UserLayout) > 0).To(BeTrue()) - }) - - Context("with unclean root path configuration", func() { - BeforeEach(func() { - config["root"] = "foo/" - }) - - It("sanitizes the root path", func() { - Expect(o.Root).To(Equal("foo")) - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/recycle.go b/pkg/storage/utils/decomposedfs/recycle.go deleted file mode 100644 index b1153674ff..0000000000 --- a/pkg/storage/utils/decomposedfs/recycle.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "os" - "path" - "path/filepath" - "strings" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// Recycle items are stored inside the node folder and start with the uuid of the deleted node. -// The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. -// The deleted file is kept in the same location/dir as the original node. This prevents deletes -// from triggering cross storage moves when the trash is accidentally stored on another partition, -// because the admin mounted a different partition there. -// TODO For an efficient listing of deleted nodes the ocis storages trash folder should have -// contain a directory with symlinks to trash files for every userid/"root" - -// ListRecycle returns the list of available recycle items. -func (fs *Decomposedfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) { - log := appctx.GetLogger(ctx) - - items := make([]*provider.RecycleItem, 0) - - // TODO how do we check if the storage allows listing the recycle for the current user? check owner of the root of the storage? - // use permissions ReadUserPermissions? - if fs.o.EnableHome { - if !node.OwnerPermissions().ListContainer { - log.Debug().Msg("owner not allowed to list trash") - return items, errtypes.PermissionDenied("owner not allowed to list trash") - } - } else { - if !node.NoPermissions().ListContainer { - log.Debug().Msg("default permissions prevent listing trash") - return items, errtypes.PermissionDenied("default permissions prevent listing trash") - } - } - - if key == "" && relativePath == "/" { - return fs.listTrashRoot(ctx) - } - - trashRoot := fs.getRecycleRoot(ctx) - f, err := os.Open(filepath.Join(trashRoot, key, relativePath)) - if err != nil { - if os.IsNotExist(err) { - return items, nil - } - return nil, errors.Wrapf(err, "tree: error listing %s", trashRoot) - } - defer f.Close() - - parentNode, err := os.Readlink(filepath.Join(trashRoot, key)) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Msg("error reading trash link, skipping") - return nil, err - } - - if md, err := f.Stat(); err != nil { - return nil, err - } else if !md.IsDir() { - // this is the case when we want to directly list a file in the trashbin - item, err := fs.createTrashItem(ctx, parentNode, filepath.Dir(relativePath), filepath.Join(trashRoot, key, relativePath)) - if err != nil { - return items, err - } - items = append(items, item) - return items, err - } - - names, err := f.Readdirnames(0) - if err != nil { - return nil, err - } - for i := range names { - if item, err := fs.createTrashItem(ctx, parentNode, relativePath, filepath.Join(trashRoot, key, relativePath, names[i])); err == nil { - items = append(items, item) - } - } - return items, nil -} - -func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, intermediatePath, itemPath string) (*provider.RecycleItem, error) { - log := appctx.GetLogger(ctx) - trashRoot := fs.getRecycleRoot(ctx) - trashnode, err := os.Readlink(itemPath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Msg("error reading trash link, skipping") - return nil, err - } - parts := strings.SplitN(filepath.Base(parentNode), ".T.", 2) - if len(parts) != 2 { - log.Error().Str("trashRoot", trashRoot).Str("trashnode", trashnode).Interface("parts", parts).Msg("malformed trash link, skipping") - return nil, errors.New("malformed trash link") - } - - nodePath := fs.lu.InternalPath(filepath.Base(trashnode)) - md, err := os.Stat(nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("trashnode", trashnode).Msg("could not stat trash item, skipping") - return nil, err - } - - item := &provider.RecycleItem{ - Type: getResourceType(md.IsDir()), - Size: uint64(md.Size()), - Key: path.Join(parts[0], intermediatePath, filepath.Base(itemPath)), - } - if deletionTime, err := time.Parse(time.RFC3339Nano, parts[1]); err == nil { - item.DeletionTime = &types.Timestamp{ - Seconds: uint64(deletionTime.Unix()), - // TODO nanos - } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("link", trashnode).Interface("parts", parts).Msg("could parse time format, ignoring") - } - - // lookup origin path in extended attributes - parentPath := fs.lu.InternalPath(filepath.Base(parentNode)) - if attrBytes, err := xattr.Get(parentPath, xattrs.TrashOriginAttr); err == nil { - item.Ref = &provider.Reference{Path: filepath.Join(string(attrBytes), intermediatePath, filepath.Base(itemPath))} - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("link", trashnode).Msg("could not read origin path, skipping") - return nil, err - } - // TODO filter results by permission ... on the original parent? or the trashed node? - // if it were on the original parent it would be possible to see files that were trashed before the current user got access - // so -> check the trash node itself - // hmm listing trash currently lists the current users trash or the 'root' trash. from ocs only the home storage is queried for trash items. - // for now we can only really check if the current user is the owner - if attrBytes, err := xattr.Get(nodePath, xattrs.OwnerIDAttr); err == nil { - if fs.o.EnableHome { - u := ctxpkg.ContextMustGetUser(ctx) - if u.Id.OpaqueId != string(attrBytes) { - log.Warn().Str("trashRoot", trashRoot).Str("link", trashnode).Msg("trash item not owned by current user, skipping") - // continue - return nil, errors.New("trash item not owned by current user") - } - } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("link", trashnode).Msg("could not read owner, skipping") - return nil, err - } - - return item, nil -} - -func (fs *Decomposedfs) listTrashRoot(ctx context.Context) ([]*provider.RecycleItem, error) { - log := appctx.GetLogger(ctx) - items := make([]*provider.RecycleItem, 0) - - trashRoot := fs.getRecycleRoot(ctx) - f, err := os.Open(trashRoot) - if err != nil { - if os.IsNotExist(err) { - return items, nil - } - return nil, errors.Wrap(err, "tree: error listing "+trashRoot) - } - defer f.Close() - - names, err := f.Readdirnames(0) - if err != nil { - return nil, err - } - - for i := range names { - trashnode, err := os.Readlink(filepath.Join(trashRoot, names[i])) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Msg("error reading trash link, skipping") - continue - } - parts := strings.SplitN(filepath.Base(trashnode), ".T.", 2) - if len(parts) != 2 { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("trashnode", trashnode).Interface("parts", parts).Msg("malformed trash link, skipping") - continue - } - - nodePath := fs.lu.InternalPath(filepath.Base(trashnode)) - md, err := os.Stat(nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("trashnode", trashnode). /*.Interface("parts", parts)*/ Msg("could not stat trash item, skipping") - continue - } - - item := &provider.RecycleItem{ - Type: getResourceType(md.IsDir()), - Size: uint64(md.Size()), - Key: parts[0], - } - if deletionTime, err := time.Parse(time.RFC3339Nano, parts[1]); err == nil { - item.DeletionTime = &types.Timestamp{ - Seconds: uint64(deletionTime.Unix()), - // TODO nanos - } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", trashnode).Interface("parts", parts).Msg("could parse time format, ignoring") - } - - // lookup origin path in extended attributes - var attrBytes []byte - if attrBytes, err = xattr.Get(nodePath, xattrs.TrashOriginAttr); err == nil { - item.Ref = &provider.Reference{Path: string(attrBytes)} - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", trashnode).Msg("could not read origin path, skipping") - continue - } - // TODO filter results by permission ... on the original parent? or the trashed node? - // if it were on the original parent it would be possible to see files that were trashed before the current user got access - // so -> check the trash node itself - // hmm listing trash currently lists the current users trash or the 'root' trash. from ocs only the home storage is queried for trash items. - // for now we can only really check if the current user is the owner - if attrBytes, err = xattr.Get(nodePath, xattrs.OwnerIDAttr); err == nil { - if fs.o.EnableHome { - u := ctxpkg.ContextMustGetUser(ctx) - if u.Id.OpaqueId != string(attrBytes) { - log.Warn().Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", trashnode).Msg("trash item not owned by current user, skipping") - continue - } - } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", trashnode).Msg("could not read owner, skipping") - continue - } - - items = append(items, item) - } - return items, nil -} - -// RestoreRecycleItem restores the specified item. -func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { - if restoreRef == nil { - restoreRef = &provider.Reference{} - } - rn, p, restoreFunc, err := fs.tp.RestoreRecycleItemFunc(ctx, key, relativePath, restoreRef.Path) - if err != nil { - return err - } - - // check permissions of deleted node - ok, err := fs.p.HasPermission(ctx, rn, func(rp *provider.ResourcePermissions) bool { - return rp.RestoreRecycleItem - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(key) - } - - ps, err := fs.p.AssemblePermissions(ctx, p) - if err != nil { - return errtypes.InternalError(err.Error()) - } - - // share receiver cannot restore to a shared resource to which she does not have write permissions. - if !ps.InitiateFileUpload { - return errtypes.PermissionDenied(key) - } - - // Run the restore func - return restoreFunc() -} - -// PurgeRecycleItem purges the specified item. -func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error { - rn, purgeFunc, err := fs.tp.PurgeRecycleItemFunc(ctx, key, relativePath) - if err != nil { - return err - } - - // check permissions of deleted node - ok, err := fs.p.HasPermission(ctx, rn, func(rp *provider.ResourcePermissions) bool { - return rp.PurgeRecycle - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(key) - } - - // Run the purge func - return purgeFunc() -} - -// EmptyRecycle empties the trash. -func (fs *Decomposedfs) EmptyRecycle(ctx context.Context) error { - u, ok := ctxpkg.ContextGetUser(ctx) - // TODO what permission should we check? we could check the root node of the user? or the owner permissions on his home root node? - // The current impl will wipe your own trash. or when no user provided the trash of 'root' - if !ok { - return os.RemoveAll(fs.getRecycleRoot(ctx)) - } - - // TODO use layout, see Tree.Delete() for problem - return os.RemoveAll(filepath.Join(fs.o.Root, "trash", u.Id.OpaqueId)) -} - -func getResourceType(isDir bool) provider.ResourceType { - if isDir { - return provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - return provider.ResourceType_RESOURCE_TYPE_FILE -} - -func (fs *Decomposedfs) getRecycleRoot(ctx context.Context) string { - if fs.o.EnableHome { - u := ctxpkg.ContextMustGetUser(ctx) - // TODO use layout, see Tree.Delete() for problem - return filepath.Join(fs.o.Root, "trash", u.Id.OpaqueId) - } - return filepath.Join(fs.o.Root, "trash", "root") -} diff --git a/pkg/storage/utils/decomposedfs/revisions.go b/pkg/storage/utils/decomposedfs/revisions.go deleted file mode 100644 index 84c00bee58..0000000000 --- a/pkg/storage/utils/decomposedfs/revisions.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "io" - "os" - "path/filepath" - "strings" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/pkg/errors" -) - -// Revision entries are stored inside the node folder and start with the same uuid as the current version. -// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions -// can be kept in the same location as the current file content. This prevents new fileuploads -// to trigger cross storage moves when revisions accidentally are stored on another partition, -// because the admin mounted a different partition there. -// We can add a background process to move old revisions to a slower storage -// and replace the revision file with a symbolic link in the future, if necessary. - -// ListRevisions lists the revisions of the given resource. -func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { - var n *node.Node - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return - } - - ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - return rp.ListFileVersions - }) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !ok: - return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - revisions = []*provider.FileVersion{} - np := n.InternalPath() - if items, err := filepath.Glob(np + ".REV.*"); err == nil { - for i := range items { - if fi, err := os.Stat(items[i]); err == nil { - mtime := fi.ModTime() - rev := &provider.FileVersion{ - Key: filepath.Base(items[i]), - Mtime: uint64(mtime.Unix()), - } - blobSize, err := node.ReadBlobSizeAttr(items[i]) - if err != nil { - return nil, errors.Wrapf(err, "error reading blobsize xattr") - } - rev.Size = uint64(blobSize) - etag, err := node.CalculateEtag(np, mtime) - if err != nil { - return nil, errors.Wrapf(err, "error calculating etag") - } - rev.Etag = etag - revisions = append(revisions, rev) - } - } - } - return -} - -// DownloadRevision returns a reader for the specified revision. -func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, ".REV.", 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") - return nil, errtypes.NotFound(revisionKey) - } - log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") - - // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, kp[0]) - if err != nil { - return nil, err - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return nil, err - } - - ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - // TODO add explicit permission in the CS3 api? - return rp.ListFileVersions && rp.RestoreFileVersion && rp.InitiateFileDownload - }) - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !ok: - return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - contentPath := fs.lu.InternalPath(revisionKey) - - r, err := os.Open(contentPath) - if err != nil { - if os.IsNotExist(err) { - return nil, errtypes.NotFound(contentPath) - } - return nil, errors.Wrap(err, "decomposedfs: error opening revision "+revisionKey) - } - return r, nil -} - -// RestoreRevision restores the specified revision of the resource. -func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (err error) { - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, ".REV.", 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") - return errtypes.NotFound(revisionKey) - } - - // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, kp[0]) - if err != nil { - return err - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - return rp.RestoreFileVersion - }) - switch { - case err != nil: - return errtypes.InternalError(err.Error()) - case !ok: - return errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - // move current version to new revision - nodePath := fs.lu.InternalPath(kp[0]) - var fi os.FileInfo - if fi, err = os.Stat(nodePath); err == nil { - // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - versionsPath := fs.lu.InternalPath(kp[0] + ".REV." + fi.ModTime().UTC().Format(time.RFC3339Nano)) - - err = os.Rename(nodePath, versionsPath) - if err != nil { - return - } - - // copy old revision to current location - - revisionPath := fs.lu.InternalPath(revisionKey) - - if err = os.Rename(revisionPath, nodePath); err != nil { - return - } - - return fs.tp.Propagate(ctx, n) - } - - log.Error().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("original node does not exist") - return -} diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go deleted file mode 100644 index d3295781af..0000000000 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "math" - "os" - "path/filepath" - "strconv" - "strings" - - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ocsconv "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/utils" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -const ( - spaceTypeAny = "*" - spaceIDAny = "*" -) - -// CreateStorageSpace creates a storage space. -func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { - // spaces will be located by default in the root of the storage. - r, err := fs.lu.RootNode(ctx) - if err != nil { - return nil, err - } - - // "everything is a resource" this is the unique ID for the Space resource. - spaceID := uuid.New().String() - - n, err := r.Child(ctx, spaceID) - if err != nil { - return nil, err - } - - if n.Exists { - return nil, fmt.Errorf("decomposedfs: spaces: invalid duplicated node with id `%s`", n.ID) - } - - if err := fs.tp.CreateDir(ctx, n); err != nil { - return nil, err - } - - // always enable propagation on the storage space root - nodePath := n.InternalPath() - // mark the space root node as the end of propagation - if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") - return nil, err - } - - if err := fs.createHiddenSpaceFolder(ctx, n); err != nil { - return nil, err - } - - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return nil, fmt.Errorf("decomposedfs: spaces: contextual user not found") - } - - if err := n.ChangeOwner(u.Id); err != nil { - return nil, err - } - - err = fs.createStorageSpace(ctx, req.Type, n.ID) - if err != nil { - return nil, err - } - - if q := req.GetQuota(); q != nil { - // set default space quota - if err := n.SetMetadata(xattrs.QuotaAttr, strconv.FormatUint(q.QuotaMaxBytes, 10)); err != nil { - return nil, err - } - } - - if err := n.SetMetadata(xattrs.SpaceNameAttr, req.Name); err != nil { - return nil, err - } - - resp := &provider.CreateStorageSpaceResponse{ - Status: &v1beta11.Status{ - Code: v1beta11.Code_CODE_OK, - }, - StorageSpace: &provider.StorageSpace{ - Owner: u, - Id: &provider.StorageSpaceId{ - OpaqueId: spaceID, - }, - // TODO we have to omit Root information because the storage driver does not know its mount point. - // Root: &provider.ResourceId{ - // StorageId: "", - // OpaqueId: "", - // }, - Name: req.GetName(), - Quota: req.GetQuota(), - SpaceType: req.GetType(), - }, - } - - nPath, err := fs.lu.Path(ctx, n) - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: spaces: could not create space. invalid node path") - } - - ctx = context.WithValue(ctx, SpaceGrant, struct{}{}) - - if err := fs.AddGrant(ctx, &provider.Reference{ - Path: nPath, - }, &provider.Grant{ - Grantee: &provider.Grantee{ - Type: provider.GranteeType_GRANTEE_TYPE_USER, - Id: &provider.Grantee_UserId{ - UserId: u.Id, - }, - }, - Permissions: ocsconv.NewManagerRole().CS3ResourcePermissions(), - }); err != nil { - return nil, err - } - - return resp, nil -} - -// ListStorageSpaces returns a list of StorageSpaces. -// The list can be filtered by space type or space id. -// Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space -// The spaceid is a concatenation of storageid + "!" + nodeid. -func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { - // TODO check filters - - // TODO when a space symlink is broken delete the space for cleanup - // read permissions are deduced from the node? - - // TODO for absolute references this actually requires us to move all user homes into a subfolder of /nodes/root, - // e.g. /nodes/root/ otherwise storage space names might collide even though they are of different types - // /nodes/root/personal/foo and /nodes/root/shares/foo might be two very different spaces, a /nodes/root/foo is not expressive enough - // we would not need /nodes/root if access always happened via spaceid+relative path - - var ( - spaceType = spaceTypeAny - spaceID = spaceIDAny - ) - - for i := range filter { - switch filter[i].Type { - case provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE: - spaceType = filter[i].GetSpaceType() - case provider.ListStorageSpacesRequest_Filter_TYPE_ID: - parts := strings.SplitN(filter[i].GetId().OpaqueId, "!", 2) - if len(parts) == 2 { - spaceID = parts[1] - } - } - } - - // build the glob path, eg. - // /path/to/root/spaces/personal/nodeid - // /path/to/root/spaces/shared/nodeid - matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceType, spaceID)) - if err != nil { - return nil, err - } - - spaces := make([]*provider.StorageSpace, 0, len(matches)) - - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - appctx.GetLogger(ctx).Debug().Msg("expected user in context") - return spaces, nil - } - - client, err := pool.GetGatewayServiceClient(pool.Endpoint(fs.o.GatewayAddr)) - if err != nil { - return nil, err - } - - checkRes, err := client.CheckPermission(ctx, &permissionsv1beta1.CheckPermissionRequest{ - Permission: "list-all-spaces", - SubjectRef: &permissionsv1beta1.SubjectReference{ - Spec: &permissionsv1beta1.SubjectReference_UserId{ - UserId: u.Id, - }, - }, - }) - if err != nil { - return nil, err - } - - canListAllSpaces := false - if checkRes.Status.Code == v1beta11.Code_CODE_OK { - canListAllSpaces = true - } - - for i := range matches { - // always read link in case storage space id != node id - if target, err := os.Readlink(matches[i]); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[i]).Msg("could not read link, skipping") - continue - } else { - n, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", filepath.Base(target)).Msg("could not read node, skipping") - continue - } - - spaceType := filepath.Base(filepath.Dir(matches[i])) - - owner, err := n.Owner() - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not read owner, skipping") - continue - } - - if spaceType == "share" && utils.UserEqual(u.Id, owner) { - // do not list shares as spaces for the owner - continue - } - - // TODO apply more filters - space, err := fs.storageSpaceFromNode(ctx, n, matches[i], spaceType, canListAllSpaces) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not convert to storage space") - continue - } - spaces = append(spaces, space) - } - } - - return spaces, nil -} - -// UpdateStorageSpace updates a storage space. -func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { - space := req.StorageSpace - - _, spaceID, err := utils.SplitStorageSpaceID(space.Id.OpaqueId) - if err != nil { - return nil, err - } - - matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceTypeAny, spaceID)) - if err != nil { - return nil, err - } - - if len(matches) != 1 { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{ - Code: v1beta11.Code_CODE_NOT_FOUND, - Message: fmt.Sprintf("update space failed: found %d matching spaces", len(matches)), - }, - }, nil - } - - target, err := os.Readlink(matches[0]) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") - } - - node, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) - if err != nil { - return nil, err - } - - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return nil, fmt.Errorf("decomposedfs: spaces: contextual user not found") - } - space.Owner = u - - if space.Name != "" { - if err := node.SetMetadata(xattrs.SpaceNameAttr, space.Name); err != nil { - return nil, err - } - } - - if space.Quota != nil { - if err := node.SetMetadata(xattrs.QuotaAttr, strconv.FormatUint(space.Quota.QuotaMaxBytes, 10)); err != nil { - return nil, err - } - } - - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_OK}, - StorageSpace: space, - }, nil -} - -// createHiddenSpaceFolder bootstraps a storage space root with a hidden ".space" folder used to store space related -// metadata such as a description or an image. -// Internally createHiddenSpaceFolder leverages the use of node.Child() to create a new node under the space root. -// createHiddenSpaceFolder is just a contextual alias for node.Child() for ".spaces". -func (fs *Decomposedfs) createHiddenSpaceFolder(ctx context.Context, r *node.Node) error { - hiddenSpace, err := r.Child(ctx, ".space") - if err != nil { - return err - } - - return fs.tp.CreateDir(ctx, hiddenSpace) -} - -func (fs *Decomposedfs) createStorageSpace(ctx context.Context, spaceType, nodeID string) error { - // create space type dir - if err := os.MkdirAll(filepath.Join(fs.o.Root, "spaces", spaceType), 0700); err != nil { - return err - } - - // we can reuse the node id as the space id - err := os.Symlink("../../nodes/"+nodeID, filepath.Join(fs.o.Root, "spaces", spaceType, nodeID)) - if err != nil { - if isAlreadyExists(err) { - appctx.GetLogger(ctx).Debug().Err(err).Str("node", nodeID).Str("spacetype", spaceType).Msg("symlink already exists") - } else { - // TODO how should we handle error cases here? - appctx.GetLogger(ctx).Error().Err(err).Str("node", nodeID).Str("spacetype", spaceType).Msg("could not create symlink") - } - } - - return nil -} - -func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Node, nodePath, spaceType string, canListAllSpaces bool) (*provider.StorageSpace, error) { - owner, err := node.Owner() - if err != nil { - return nil, err - } - - // TODO apply more filters - - sname, err := xattr.Get(node.InternalPath(), xattrs.SpaceNameAttr) - if err != nil { - return nil, err - } - space := &provider.StorageSpace{ - // FIXME the driver should know its id move setting the spaceid from the storage provider to the drivers - //Id: &provider.StorageSpaceId{OpaqueId: "1284d238-aa92-42ce-bdc4-0b0000009157!" + n.ID}, - Root: &provider.ResourceId{ - // FIXME the driver should know its id move setting the spaceid from the storage provider to the drivers - //StorageId: "1284d238-aa92-42ce-bdc4-0b0000009157", - OpaqueId: node.ID, - }, - Name: string(sname), - SpaceType: spaceType, - // Mtime is set either as node.tmtime or as fi.mtime below - } - - user := ctxpkg.ContextMustGetUser(ctx) - - // filter out spaces user cannot access (currently based on stat permission) - if !canListAllSpaces { - p, err := node.ReadUserPermissions(ctx, user) - if err != nil { - return nil, err - } - if !p.Stat { - return nil, errors.New("user is not allowed to Stat the space") - } - } - - space.Owner = &userv1beta1.User{ // FIXME only return a UserID, not a full blown user object - Id: owner, - } - - // we set the space mtime to the root item mtime - // override the stat mtime with a tmtime if it is present - if tmt, err := node.GetTMTime(); err == nil { - un := tmt.UnixNano() - space.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - } else if fi, err := os.Stat(nodePath); err == nil { - // fall back to stat mtime - un := fi.ModTime().UnixNano() - space.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - } - - // quota - v, err := xattr.Get(nodePath, xattrs.QuotaAttr) - if err == nil { - // make sure we have a proper signed int - // we use the same magic numbers to indicate: - // -1 = uncalculated - // -2 = unknown - // -3 = unlimited - if quota, err := strconv.ParseUint(string(v), 10, 64); err == nil { - space.Quota = &provider.Quota{ - QuotaMaxBytes: quota, - QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited? - } - } else { - return nil, err - } - } - - return space, nil -} diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go deleted file mode 100644 index a80f3e3755..0000000000 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package helpers - -import ( - "context" - "os" - "path/filepath" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ruser "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/mocks" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree" - treemocks "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree/mocks" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/tests/helpers" - "github.com/google/uuid" - "github.com/pkg/xattr" - "github.com/stretchr/testify/mock" -) - -// TestEnv represents a test environment for unit tests. -type TestEnv struct { - Root string - Fs storage.FS - Tree *tree.Tree - Permissions *mocks.PermissionsChecker - Blobstore *treemocks.Blobstore - Owner *userpb.User - Lookup *decomposedfs.Lookup - Ctx context.Context -} - -// NewTestEnv prepares a test environment on disk -// The storage contains some directories and a file: -// -// /dir1/ -// /dir1/file1 -// /dir1/subdir1/ -func NewTestEnv() (*TestEnv, error) { - tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") - if err != nil { - return nil, err - } - - config := map[string]interface{}{ - "root": tmpRoot, - "enable_home": true, - "treetime_accounting": true, - "treesize_accounting": true, - "share_folder": "/Shares", - "user_layout": "{{.Id.OpaqueId}}", - } - o, err := options.New(config) - if err != nil { - return nil, err - } - - owner := &userpb.User{ - Id: &userpb.UserId{ - Idp: "idp", - OpaqueId: "userid", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - Username: "username", - } - lookup := &decomposedfs.Lookup{Options: o} - permissions := &mocks.PermissionsChecker{} - permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Times(3) // Permissions required for setup below - bs := &treemocks.Blobstore{} - tree := tree.New(o.Root, true, true, lookup, bs) - fs, err := decomposedfs.New(o, lookup, permissions, tree) - if err != nil { - return nil, err - } - ctx := ruser.ContextSetUser(context.Background(), owner) - - env := &TestEnv{ - Root: tmpRoot, - Fs: fs, - Tree: tree, - Lookup: lookup, - Permissions: permissions, - Blobstore: bs, - Owner: owner, - Ctx: ctx, - } - - // Create home - err = fs.CreateHome(ctx) - if err != nil { - return nil, err - } - - // the space name attribute is the stop condition in the lookup - h, err := lookup.HomeNode(ctx) - if err != nil { - return nil, err - } - if err = xattr.Set(h.InternalPath(), xattrs.SpaceNameAttr, []byte("username")); err != nil { - return nil, err - } - - // Create dir1 - dir1, err := env.CreateTestDir("/dir1") - if err != nil { - return nil, err - } - - // Create file1 in dir1 - _, err = env.CreateTestFile("file1", "file1-blobid", 1234, dir1.ID) - if err != nil { - return nil, err - } - - // Create subdir1 in dir1 - err = fs.CreateDir(ctx, &providerv1beta1.Reference{Path: "/dir1/subdir1"}) - if err != nil { - return nil, err - } - - dir2, err := dir1.Child(ctx, "subdir1") - if err != nil { - return nil, err - } - // Create file1 in dir1 - _, err = env.CreateTestFile("file2", "file2-blobid", 12345, dir2.ID) - if err != nil { - return nil, err - } - - // Create emptydir - err = fs.CreateDir(ctx, &providerv1beta1.Reference{Path: "/emptydir"}) - if err != nil { - return nil, err - } - - return env, nil -} - -// Cleanup removes all files from disk. -func (t *TestEnv) Cleanup() { - os.RemoveAll(t.Root) -} - -// CreateTestDir create a directory and returns a corresponding Node. -func (t *TestEnv) CreateTestDir(name string) (*node.Node, error) { - err := t.Fs.CreateDir(t.Ctx, &providerv1beta1.Reference{Path: name}) - if err != nil { - return nil, err - } - n, err := t.Lookup.NodeFromPath(t.Ctx, name, false) - if err != nil { - return nil, err - } - - return n, nil -} - -// CreateTestFile creates a new file and its metadata and returns a corresponding Node. -func (t *TestEnv) CreateTestFile(name, blobID string, blobSize int64, parentID string) (*node.Node, error) { - // Create file in dir1 - file := node.New( - uuid.New().String(), - parentID, - name, - blobSize, - blobID, - nil, - t.Lookup, - ) - _, err := os.OpenFile(file.InternalPath(), os.O_CREATE, 0700) - if err != nil { - return nil, err - } - err = file.WriteMetadata(t.Owner.Id) - if err != nil { - return nil, err - } - // Link in parent - childNameLink := filepath.Join(t.Lookup.InternalPath(file.ParentID), file.Name) - err = os.Symlink("../"+file.ID, childNameLink) - if err != nil { - return nil, err - } - - return file, err -} diff --git a/pkg/storage/utils/decomposedfs/tree/mocks/Blobstore.go b/pkg/storage/utils/decomposedfs/tree/mocks/Blobstore.go deleted file mode 100644 index 46c4aaa37b..0000000000 --- a/pkg/storage/utils/decomposedfs/tree/mocks/Blobstore.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - io "io" - - mock "github.com/stretchr/testify/mock" -) - -// Blobstore is an autogenerated mock type for the Blobstore type -type Blobstore struct { - mock.Mock -} - -// Delete provides a mock function with given fields: key -func (_m *Blobstore) Delete(key string) error { - ret := _m.Called(key) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(key) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Download provides a mock function with given fields: key -func (_m *Blobstore) Download(key string) (io.ReadCloser, error) { - ret := _m.Called(key) - - var r0 io.ReadCloser - if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { - r0 = rf(key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.ReadCloser) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(key) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Upload provides a mock function with given fields: key, reader -func (_m *Blobstore) Upload(key string, reader io.Reader) error { - ret := _m.Called(key, reader) - - var r0 error - if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(key, reader) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go deleted file mode 100644 index 74f8edd62a..0000000000 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ /dev/null @@ -1,867 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package tree - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/pkg/utils" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/pkg/xattr" - "github.com/rs/zerolog/log" -) - -// go:generate mockery -name Blobstore - -const ( - spaceTypePersonal = "personal" - spaceTypeShare = "share" -) - -// Blobstore defines an interface for storing blobs in a blobstore. -type Blobstore interface { - Upload(key string, reader io.Reader) error - Download(key string) (io.ReadCloser, error) - Delete(key string) error -} - -// PathLookup defines the interface for the lookup component. -type PathLookup interface { - NodeFromPath(ctx context.Context, fn string, followReferences bool) (*node.Node, error) - NodeFromID(ctx context.Context, id *provider.ResourceId) (n *node.Node, err error) - RootNode(ctx context.Context) (node *node.Node, err error) - HomeOrRootNode(ctx context.Context) (node *node.Node, err error) - - InternalRoot() string - InternalPath(ID string) string - Path(ctx context.Context, n *node.Node) (path string, err error) - ShareFolder() string -} - -// Tree manages a hierarchical tree. -type Tree struct { - lookup PathLookup - blobstore Blobstore - - root string - treeSizeAccounting bool - treeTimeAccounting bool -} - -// PermissionCheckFunc defined a function used to check resource permissions. -type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool - -// New returns a new instance of Tree. -func New(root string, tta bool, tsa bool, lu PathLookup, bs Blobstore) *Tree { - return &Tree{ - lookup: lu, - blobstore: bs, - root: root, - treeTimeAccounting: tta, - treeSizeAccounting: tsa, - } -} - -// Setup prepares the tree structure. -func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { - // create data paths for internal layout - dataPaths := []string{ - filepath.Join(t.root, "nodes"), - // notes contain symlinks from nodes//uploads/ to ../../uploads/ - // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir - filepath.Join(t.root, "uploads"), - filepath.Join(t.root, "trash"), - } - for _, v := range dataPaths { - err := os.MkdirAll(v, 0700) - if err != nil { - return err - } - } - - // the root node has an empty name - // the root node has no parent - n := node.New("root", "", "", 0, "", nil, t.lookup) - err := t.createNode(n, owner) - if err != nil { - return err - } - - // set propagation flag - v := []byte("0") - if propagateToRoot { - v = []byte("1") - } - if err = xattr.Set(n.InternalPath(), xattrs.PropagationAttr, v); err != nil { - return err - } - - // create spaces folder and iterate over existing nodes to populate it - spacesPath := filepath.Join(t.root, "spaces") - fi, err := os.Stat(spacesPath) - if os.IsNotExist(err) { - // create personal spaces dir - if err := os.MkdirAll(filepath.Join(spacesPath, spaceTypePersonal), 0700); err != nil { - return err - } - // create share spaces dir - if err := os.MkdirAll(filepath.Join(spacesPath, spaceTypeShare), 0700); err != nil { - return err - } - - f, err := os.Open(filepath.Join(t.root, "nodes")) - if err != nil { - return err - } - nodes, err := f.Readdir(0) - if err != nil { - return err - } - - for i := range nodes { - nodePath := filepath.Join(t.root, "nodes", nodes[i].Name()) - - // is it a user root? -> create personal space - if isRootNode(nodePath) { - // we can reuse the node id as the space id - t.linkSpace(spaceTypePersonal, nodes[i].Name(), nodes[i].Name()) - } - - // is it a shared node? -> create share space - if isSharedNode(nodePath) { - // we can reuse the node id as the space id - t.linkSpace(spaceTypeShare, nodes[i].Name(), nodes[i].Name()) - } - } - } else if !fi.IsDir() { - // check if it is a directory - return fmt.Errorf("%s is not a directory", spacesPath) - } - - return nil -} - -// linkSpace creates a new symbolic link for a space with the given type st, and node id. -func (t *Tree) linkSpace(spaceType, spaceID, nodeID string) { - spacesPath := filepath.Join(t.root, "spaces", spaceType, spaceID) - expectedTarget := "../../nodes/" + nodeID - linkTarget, err := os.Readlink(spacesPath) - if errors.Is(err, os.ErrNotExist) { - err = os.Symlink(expectedTarget, spacesPath) - if err != nil { - logger.New().Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Str("node", nodeID). - Msg("could not create symlink") - } - } else { - if err != nil { - logger.New().Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Str("node", nodeID). - Msg("could not read symlink") - } - if linkTarget != expectedTarget { - logger.New().Warn(). - Str("space_type", spaceType). - Str("space", spaceID). - Str("node", nodeID). - Str("expected", expectedTarget). - Str("actual", linkTarget). - Msg("expected a different link target") - } - } -} - -func isRootNode(nodePath string) bool { - attrBytes, err := xattr.Get(nodePath, xattrs.ParentidAttr) - return err == nil && string(attrBytes) == "root" -} -func isSharedNode(nodePath string) bool { - if attrs, err := xattr.List(nodePath); err == nil { - for i := range attrs { - if strings.HasPrefix(attrs[i], xattrs.GrantPrefix) { - return true - } - } - } - return false -} - -// GetMD returns the metadata of a node in the tree. -func (t *Tree) GetMD(ctx context.Context, n *node.Node) (os.FileInfo, error) { - md, err := os.Stat(n.InternalPath()) - if err != nil { - if os.IsNotExist(err) { - return nil, errtypes.NotFound(n.ID) - } - return nil, errors.Wrap(err, "tree: error stating "+n.ID) - } - - return md, nil -} - -// CreateDir creates a new directory entry in the tree. -func (t *Tree) CreateDir(ctx context.Context, n *node.Node) (err error) { - if n.Exists || n.ID != "" { - return errtypes.AlreadyExists(n.ID) // path? - } - - // create a directory node - n.ID = uuid.New().String() - - // who will become the owner? the owner of the parent node, not the current user - var p *node.Node - p, err = n.Parent() - if err != nil { - return - } - var owner *userpb.UserId - owner, err = p.Owner() - if err != nil { - return - } - - err = t.createNode(n, owner) - if err != nil { - return nil - } - - // make child appear in listings - err = os.Symlink("../"+n.ID, filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name)) - if err != nil { - return - } - return t.Propagate(ctx, n) -} - -// Move replaces the target with the source. -func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error) { - // if target exists delete it without trashing it - if newNode.Exists { - // TODO make sure all children are deleted - if err := os.RemoveAll(newNode.InternalPath()); err != nil { - return errors.Wrap(err, "decomposedfs: Move: error deleting target node "+newNode.ID) - } - } - - // Always target the old node ID for xattr updates. - // The new node id is empty if the target does not exist - // and we need to overwrite the new one when overwriting an existing path. - tgtPath := oldNode.InternalPath() - - // are we just renaming (parent stays the same)? - if oldNode.ParentID == newNode.ParentID { - parentPath := t.lookup.InternalPath(oldNode.ParentID) - - // rename child - err = os.Rename( - filepath.Join(parentPath, oldNode.Name), - filepath.Join(parentPath, newNode.Name), - ) - if err != nil { - return errors.Wrap(err, "decomposedfs: could not rename child") - } - - // update name attribute - if err := xattr.Set(tgtPath, xattrs.NameAttr, []byte(newNode.Name)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set name attribute") - } - - return t.Propagate(ctx, newNode) - } - - // we are moving the node to a new parent, any target has been removed - // bring old node to the new parent - - // rename child - err = os.Rename( - filepath.Join(t.lookup.InternalPath(oldNode.ParentID), oldNode.Name), - filepath.Join(t.lookup.InternalPath(newNode.ParentID), newNode.Name), - ) - if err != nil { - return errors.Wrap(err, "decomposedfs: could not move child") - } - - // update target parentid and name - if err := xattr.Set(tgtPath, xattrs.ParentidAttr, []byte(newNode.ParentID)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set parentid attribute") - } - if err := xattr.Set(tgtPath, xattrs.NameAttr, []byte(newNode.Name)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set name attribute") - } - - // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? - // collect in a list, then only stat each node once - // also do this in a go routine ... webdav should check the etag async - - err = t.Propagate(ctx, oldNode) - if err != nil { - return errors.Wrap(err, "decomposedfs: Move: could not propagate old node") - } - err = t.Propagate(ctx, newNode) - if err != nil { - return errors.Wrap(err, "decomposedfs: Move: could not propagate new node") - } - return nil -} - -// ListFolder lists the content of a folder node. -func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, error) { - dir := n.InternalPath() - f, err := os.Open(dir) - if err != nil { - if os.IsNotExist(err) { - return nil, errtypes.NotFound(dir) - } - return nil, errors.Wrap(err, "tree: error listing "+dir) - } - defer f.Close() - - names, err := f.Readdirnames(0) - if err != nil { - return nil, err - } - nodes := []*node.Node{} - for i := range names { - link, err := os.Readlink(filepath.Join(dir, names[i])) - if err != nil { - // TODO log - continue - } - - n, err := node.ReadNode(ctx, t.lookup, filepath.Base(link)) - if err != nil { - // TODO log - continue - } - nodes = append(nodes, n) - } - return nodes, nil -} - -// Delete deletes a node in the tree by moving it to the trash. -func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { - deletingSharedResource := ctx.Value(appctx.DeletingSharedResource) - - if deletingSharedResource != nil && deletingSharedResource.(bool) { - src := filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name) - return os.Remove(src) - } - // Prepare the trash - // TODO use layout?, but it requires resolving the owners user if the username is used instead of the id. - // the node knows the owner id so we use that for now - o, err := n.Owner() - if err != nil { - return - } - if o.OpaqueId == "" { - // fall back to root trash - o.OpaqueId = "root" - } - err = os.MkdirAll(filepath.Join(t.root, "trash", o.OpaqueId), 0700) - if err != nil { - return - } - - // get the original path - origin, err := t.lookup.Path(ctx, n) - if err != nil { - return - } - - // set origin location in metadata - nodePath := n.InternalPath() - if err := xattr.Set(nodePath, xattrs.TrashOriginAttr, []byte(origin)); err != nil { - return err - } - - deletionTime := time.Now().UTC().Format(time.RFC3339Nano) - - // first make node appear in the owners (or root) trash - // parent id and name are stored as extended attributes in the node itself - trashLink := filepath.Join(t.root, "trash", o.OpaqueId, n.ID) - err = os.Symlink("../../nodes/"+n.ID+".T."+deletionTime, trashLink) - if err != nil { - // To roll back changes - // TODO unset trashOriginAttr - return - } - - // at this point we have a symlink pointing to a non existing destination, which is fine - - // rename the trashed node so it is not picked up when traversing up the tree and matches the symlink - trashPath := nodePath + ".T." + deletionTime - err = os.Rename(nodePath, trashPath) - if err != nil { - // To roll back changes - // TODO remove symlink - // TODO unset trashOriginAttr - return - } - - // finally remove the entry from the parent dir - src := filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name) - err = os.Remove(src) - if err != nil { - // To roll back changes - // TODO revert the rename - // TODO remove symlink - // TODO unset trashOriginAttr - return - } - - return t.Propagate(ctx, n) -} - -// RestoreRecycleItemFunc returns a node and a function to restore it from the trash. -func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, key, trashPath, restorePath string) (*node.Node, *node.Node, func() error, error) { - rn, trashItem, deletedNodePath, origin, err := t.readRecycleItem(ctx, key, trashPath) - if err != nil { - return nil, nil, nil, err - } - - if restorePath == "" { - restorePath = origin - } - - var target *node.Node - target, err = t.lookup.NodeFromPath(ctx, restorePath, true) - if err != nil { - return nil, nil, nil, err - } - - p, err := target.Parent() - if err != nil { - return nil, nil, nil, err - } - - fn := func() error { - // link to origin - var n *node.Node - n, err = t.lookup.NodeFromPath(ctx, restorePath, true) - if err != nil { - return err - } - if n.Exists { - return errtypes.AlreadyExists("origin already exists") - } - - // add the entry for the parent dir - err = os.Symlink("../"+rn.ID, filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name)) - if err != nil { - return err - } - - // rename to node only name, so it is picked up by id - nodePath := rn.InternalPath() - - // attempt to rename only if we're not in a subfolder - if deletedNodePath != nodePath { - err = os.Rename(deletedNodePath, nodePath) - if err != nil { - return err - } - } - - // the new node will inherit the permissions of its parent - p, err := n.Parent() - if err != nil { - return err - } - - po, err := p.Owner() - if err != nil { - return err - } - - if err := rn.ChangeOwner(po); err != nil { - return err - } - - n.Exists = true - // update name attribute - if err := xattr.Set(nodePath, xattrs.NameAttr, []byte(n.Name)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set name attribute") - } - - // set ParentidAttr to restorePath's node parent id - if trashPath != "" { - if err := xattr.Set(nodePath, xattrs.ParentidAttr, []byte(n.ParentID)); err != nil { - return errors.Wrap(err, "decomposedfs: could not set name attribute") - } - } - - // delete item link in trash - if err = os.Remove(trashItem); err != nil { - log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trashitem") - } - return t.Propagate(ctx, n) - } - return rn, p, fn, nil -} - -// PurgeRecycleItemFunc returns a node and a function to purge it from the trash. -func (t *Tree) PurgeRecycleItemFunc(ctx context.Context, key string, path string) (*node.Node, func() error, error) { - rn, trashItem, deletedNodePath, _, err := t.readRecycleItem(ctx, key, path) - if err != nil { - return nil, nil, err - } - - fn := func() error { - if err := os.RemoveAll(deletedNodePath); err != nil { - log.Error().Err(err).Str("deletedNodePath", deletedNodePath).Msg("error deleting trash node") - return err - } - - // delete blob from blobstore - if rn.BlobID != "" { - if err = t.DeleteBlob(rn.BlobID); err != nil { - log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trash item blob") - return err - } - } - - // delete item link in trash - if err = os.Remove(trashItem); err != nil { - log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trash item") - return err - } - - return nil - } - - return rn, fn, nil -} - -// Propagate propagates changes to the root of the tree. -func (t *Tree) Propagate(ctx context.Context, n *node.Node) (err error) { - sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - if !t.treeTimeAccounting && !t.treeSizeAccounting { - // no propagation enabled - sublog.Debug().Msg("propagation disabled") - return - } - - // is propagation enabled for the parent node? - - var root *node.Node - if n.SpaceRoot == nil { - if root, err = t.lookup.HomeOrRootNode(ctx); err != nil { - return - } - } else { - root = n.SpaceRoot - } - - // use a sync time and don't rely on the mtime of the current node, as the stat might not change when a rename happened too quickly - sTime := time.Now().UTC() - - // we loop until we reach the root - for err == nil && n.ID != root.ID { - sublog.Debug().Msg("propagating") - - if n, err = n.Parent(); err != nil { - break - } - - sublog = sublog.With().Interface("node", n).Logger() - - // TODO none, sync and async? - if !n.HasPropagation() { - sublog.Debug().Str("attr", xattrs.PropagationAttr).Msg("propagation attribute not set or unreadable, not propagating") - // if the attribute is not set treat it as false / none / no propagation - return nil - } - - if t.treeTimeAccounting { - // update the parent tree time if it is older than the nodes mtime - updateSyncTime := false - - var tmTime time.Time - tmTime, err = n.GetTMTime() - switch { - case err != nil: - // missing attribute, or invalid format, overwrite - sublog.Debug().Err(err). - Msg("could not read tmtime attribute, overwriting") - updateSyncTime = true - case tmTime.Before(sTime): - sublog.Debug(). - Time("tmtime", tmTime). - Time("stime", sTime). - Msg("parent tmtime is older than node mtime, updating") - updateSyncTime = true - default: - sublog.Debug(). - Time("tmtime", tmTime). - Time("stime", sTime). - Dur("delta", sTime.Sub(tmTime)). - Msg("parent tmtime is younger than node mtime, not updating") - } - - if updateSyncTime { - // update the tree time of the parent node - if err = n.SetTMTime(sTime); err != nil { - sublog.Error().Err(err).Time("tmtime", sTime).Msg("could not update tmtime of parent node") - } else { - sublog.Debug().Time("tmtime", sTime).Msg("updated tmtime of parent node") - } - } - - if err := n.UnsetTempEtag(); err != nil { - sublog.Error().Err(err).Msg("could not remove temporary etag attribute") - } - } - - // size accounting - if t.treeSizeAccounting { - // update the treesize if it differs from the current size - updateTreeSize := false - - var treeSize, calculatedTreeSize uint64 - calculatedTreeSize, err = calculateTreeSize(ctx, n.InternalPath()) - if err != nil { - continue - } - - treeSize, err = n.GetTreeSize() - switch { - case err != nil: - // missing attribute, or invalid format, overwrite - sublog.Debug().Err(err).Msg("could not read treesize attribute, overwriting") - updateTreeSize = true - case treeSize != calculatedTreeSize: - sublog.Debug(). - Uint64("treesize", treeSize). - Uint64("calculatedTreeSize", calculatedTreeSize). - Msg("parent treesize is different then calculated treesize, updating") - updateTreeSize = true - default: - sublog.Debug(). - Uint64("treesize", treeSize). - Uint64("calculatedTreeSize", calculatedTreeSize). - Msg("parent size matches calculated size, not updating") - } - - if updateTreeSize { - // update the tree time of the parent node - if err = n.SetTreeSize(calculatedTreeSize); err != nil { - sublog.Error().Err(err).Uint64("calculatedTreeSize", calculatedTreeSize).Msg("could not update treesize of parent node") - } else { - sublog.Debug().Uint64("calculatedTreeSize", calculatedTreeSize).Msg("updated treesize of parent node") - } - } - } - } - if err != nil { - sublog.Error().Err(err).Msg("error propagating") - return - } - return -} - -func calculateTreeSize(ctx context.Context, nodePath string) (uint64, error) { - var size uint64 - - f, err := os.Open(nodePath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not open dir") - return 0, err - } - defer f.Close() - - names, err := f.Readdirnames(0) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read dirnames") - return 0, err - } - for i := range names { - cPath := filepath.Join(nodePath, names[i]) - info, err := os.Stat(cPath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not stat child entry") - continue // continue after an error - } - if !info.IsDir() { - blobSize, err := node.ReadBlobSizeAttr(cPath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not read blobSize xattr") - continue // continue after an error - } - size += uint64(blobSize) - } else { - // read from attr - var b []byte - // xattr.Get will follow the symlink - if b, err = xattr.Get(cPath, xattrs.TreesizeAttr); err != nil { - // TODO recursively descend and recalculate treesize - continue // continue after an error - } - csize, err := strconv.ParseUint(string(b), 10, 64) - if err != nil { - // TODO recursively descend and recalculate treesize - continue // continue after an error - } - size += csize - } - } - return size, err -} - -// WriteBlob writes a blob to the blobstore. -func (t *Tree) WriteBlob(key string, reader io.Reader) error { - return t.blobstore.Upload(key, reader) -} - -// ReadBlob reads a blob from the blobstore. -func (t *Tree) ReadBlob(key string) (io.ReadCloser, error) { - return t.blobstore.Download(key) -} - -// DeleteBlob deletes a blob from the blobstore. -func (t *Tree) DeleteBlob(key string) error { - if key == "" { - return fmt.Errorf("could not delete blob, empty key was given") - } - - return t.blobstore.Delete(key) -} - -// TODO check if node exists? -func (t *Tree) createNode(n *node.Node, owner *userpb.UserId) (err error) { - // create a directory node - nodePath := n.InternalPath() - if err = os.MkdirAll(nodePath, 0700); err != nil { - return errors.Wrap(err, "decomposedfs: error creating node") - } - - return n.WriteMetadata(owner) -} - -// TODO refactor the returned params into Node properties? would make all the path transformations go away... -func (t *Tree) readRecycleItem(ctx context.Context, key, path string) (n *node.Node, trashItem string, deletedNodePath string, origin string, err error) { - if key == "" { - return nil, "", "", "", errtypes.InternalError("key is empty") - } - - u := ctxpkg.ContextMustGetUser(ctx) - trashItem = filepath.Join(t.lookup.InternalRoot(), "trash", u.Id.OpaqueId, key, path) - - var link string - link, err = os.Readlink(trashItem) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Msg("error reading trash link") - return - } - - nodeID := filepath.Base(link) - if path == "" || path == "/" { - parts := strings.SplitN(filepath.Base(link), ".T.", 2) - if len(parts) != 2 { - appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Interface("parts", parts).Msg("malformed trash link") - return - } - nodeID = parts[0] - } - - var attrBytes []byte - deletedNodePath = t.lookup.InternalPath(filepath.Base(link)) - - owner := &userpb.UserId{} - // lookup ownerId in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerIDAttr); err == nil { - owner.OpaqueId = string(attrBytes) - } else { - return - } - // lookup ownerIdp in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerIDPAttr); err == nil { - owner.Idp = string(attrBytes) - } else { - return - } - // lookup ownerType in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerTypeAttr); err == nil { - owner.Type = utils.UserTypeMap(string(attrBytes)) - } else { - return - } - - n = node.New(nodeID, "", "", 0, "", owner, t.lookup) - // lookup blobID in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.BlobIDAttr); err == nil { - n.BlobID = string(attrBytes) - } else { - return - } - - // lookup parent id in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.ParentidAttr); err == nil { - n.ParentID = string(attrBytes) - } else { - return - } - // lookup name in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.NameAttr); err == nil { - n.Name = string(attrBytes) - } else { - return - } - - // get origin node - origin = "/" - - deletedNodeRootPath := deletedNodePath - if path != "" && path != "/" { - trashItemRoot := filepath.Join(t.lookup.InternalRoot(), "trash", u.Id.OpaqueId, key) - var rootLink string - rootLink, err = os.Readlink(trashItemRoot) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Msg("error reading trash link") - return - } - deletedNodeRootPath = t.lookup.InternalPath(filepath.Base(rootLink)) - } - // lookup origin path in extended attributes - if attrBytes, err = xattr.Get(deletedNodeRootPath, xattrs.TrashOriginAttr); err == nil { - origin = filepath.Join(string(attrBytes), path) - } else { - log.Error().Err(err).Str("trashItem", trashItem).Str("link", link).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") - } - return -} diff --git a/pkg/storage/utils/decomposedfs/tree/tree_suite_test.go b/pkg/storage/utils/decomposedfs/tree/tree_suite_test.go deleted file mode 100644 index a811bc03fa..0000000000 --- a/pkg/storage/utils/decomposedfs/tree/tree_suite_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package tree_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestTree(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Tree Suite") -} diff --git a/pkg/storage/utils/decomposedfs/tree/tree_test.go b/pkg/storage/utils/decomposedfs/tree/tree_test.go deleted file mode 100644 index 9558608357..0000000000 --- a/pkg/storage/utils/decomposedfs/tree/tree_test.go +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package tree_test - -import ( - "os" - "path" - - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/xattr" - "github.com/stretchr/testify/mock" -) - -var _ = Describe("Tree", func() { - var ( - env *helpers.TestEnv - - t *tree.Tree - ) - - JustBeforeEach(func() { - var err error - env, err = helpers.NewTestEnv() - Expect(err).ToNot(HaveOccurred()) - t = env.Tree - }) - - AfterEach(func() { - if env != nil { - env.Cleanup() - } - }) - - Context("with an existingfile", func() { - var ( - n *node.Node - originalPath = "dir1/file1" - ) - - JustBeforeEach(func() { - var err error - n, err = env.Lookup.NodeFromPath(env.Ctx, originalPath, false) - Expect(err).ToNot(HaveOccurred()) - }) - - Describe("Delete", func() { - JustBeforeEach(func() { - _, err := os.Stat(n.InternalPath()) - Expect(err).ToNot(HaveOccurred()) - - Expect(t.Delete(env.Ctx, n)).To(Succeed()) - - _, err = os.Stat(n.InternalPath()) - Expect(err).To(HaveOccurred()) - }) - - It("moves the file to the trash", func() { - trashPath := path.Join(env.Root, "trash", env.Owner.Id.OpaqueId, n.ID) - _, err := os.Stat(trashPath) - Expect(err).ToNot(HaveOccurred()) - }) - - It("removes the file from its original location", func() { - _, err := os.Stat(n.InternalPath()) - Expect(err).To(HaveOccurred()) - }) - - It("sets the trash origin xattr", func() { - trashPath := path.Join(env.Root, "trash", env.Owner.Id.OpaqueId, n.ID) - attr, err := xattr.Get(trashPath, xattrs.TrashOriginAttr) - Expect(err).ToNot(HaveOccurred()) - Expect(string(attr)).To(Equal("/dir1/file1")) - }) - - It("does not delete the blob from the blobstore", func() { - env.Blobstore.AssertNotCalled(GinkgoT(), "Delete", mock.AnythingOfType("string")) - }) - }) - - Context("that was deleted", func() { - var ( - trashPath string - ) - - JustBeforeEach(func() { - env.Blobstore.On("Delete", n.BlobID).Return(nil) - trashPath = path.Join(env.Root, "trash", env.Owner.Id.OpaqueId, n.ID) - Expect(t.Delete(env.Ctx, n)).To(Succeed()) - }) - - Describe("PurgeRecycleItemFunc", func() { - JustBeforeEach(func() { - _, err := os.Stat(trashPath) - Expect(err).ToNot(HaveOccurred()) - - _, purgeFunc, err := t.PurgeRecycleItemFunc(env.Ctx, n.ID, "") - Expect(err).ToNot(HaveOccurred()) - Expect(purgeFunc()).To(Succeed()) - }) - - It("removes the file from the trash", func() { - _, err := os.Stat(trashPath) - Expect(err).To(HaveOccurred()) - }) - - It("deletes the blob from the blobstore", func() { - env.Blobstore.AssertCalled(GinkgoT(), "Delete", mock.AnythingOfType("string")) - }) - }) - - Describe("RestoreRecycleItemFunc", func() { - JustBeforeEach(func() { - _, err := os.Stat(trashPath) - Expect(err).ToNot(HaveOccurred()) - _, err = os.Stat(n.InternalPath()) - Expect(err).To(HaveOccurred()) - }) - - It("restores the file to its original location if the targetPath is empty", func() { - _, _, restoreFunc, err := t.RestoreRecycleItemFunc(env.Ctx, n.ID, "", "") - Expect(err).ToNot(HaveOccurred()) - - Expect(restoreFunc()).To(Succeed()) - - originalNode, err := env.Lookup.NodeFromPath(env.Ctx, originalPath, false) - Expect(err).ToNot(HaveOccurred()) - Expect(originalNode.Exists).To(BeTrue()) - }) - - It("restores files to different locations", func() { - _, _, restoreFunc, err := t.RestoreRecycleItemFunc(env.Ctx, n.ID, "", "dir1/newLocation") - Expect(err).ToNot(HaveOccurred()) - - Expect(restoreFunc()).To(Succeed()) - - newNode, err := env.Lookup.NodeFromPath(env.Ctx, "dir1/newLocation", false) - Expect(err).ToNot(HaveOccurred()) - Expect(newNode.Exists).To(BeTrue()) - - originalNode, err := env.Lookup.NodeFromPath(env.Ctx, originalPath, false) - Expect(err).ToNot(HaveOccurred()) - Expect(originalNode.Exists).To(BeFalse()) - }) - - It("removes the file from the trash", func() { - _, _, restoreFunc, err := t.RestoreRecycleItemFunc(env.Ctx, n.ID, "", "") - Expect(err).ToNot(HaveOccurred()) - - Expect(restoreFunc()).To(Succeed()) - - _, err = os.Stat(trashPath) - Expect(err).To(HaveOccurred()) - }) - }) - }) - }) - - Context("with an empty directory", func() { - var ( - n *node.Node - ) - - JustBeforeEach(func() { - var err error - n, err = env.Lookup.NodeFromPath(env.Ctx, "emptydir", false) - Expect(err).ToNot(HaveOccurred()) - }) - - Context("that was deleted", func() { - var ( - trashPath string - ) - - JustBeforeEach(func() { - trashPath = path.Join(env.Root, "trash", env.Owner.Id.OpaqueId, n.ID) - Expect(t.Delete(env.Ctx, n)).To(Succeed()) - }) - - Describe("PurgeRecycleItemFunc", func() { - JustBeforeEach(func() { - _, err := os.Stat(trashPath) - Expect(err).ToNot(HaveOccurred()) - - _, purgeFunc, err := t.PurgeRecycleItemFunc(env.Ctx, n.ID, "") - Expect(err).ToNot(HaveOccurred()) - Expect(purgeFunc()).To(Succeed()) - }) - - It("removes the file from the trash", func() { - _, err := os.Stat(trashPath) - Expect(err).To(HaveOccurred()) - }) - - It("does not try to delete a blob from the blobstore", func() { - env.Blobstore.AssertNotCalled(GinkgoT(), "Delete", mock.AnythingOfType("string")) - }) - }) - }) - }) - - Describe("Propagate", func() { - var dir *node.Node - - JustBeforeEach(func() { - env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - - // Create test dir - var err error - dir, err = env.CreateTestDir("testdir") - Expect(err).ToNot(HaveOccurred()) - }) - - Describe("with TreeTimeAccounting enabled", func() { - It("sets the tmtime of the parent", func() { - file, err := env.CreateTestFile("file1", "", 1, dir.ID) - Expect(err).ToNot(HaveOccurred()) - - perms := node.OwnerPermissions() - riBefore, err := dir.AsResourceInfo(env.Ctx, &perms, []string{}, false) - Expect(err).ToNot(HaveOccurred()) - - err = env.Tree.Propagate(env.Ctx, file) - Expect(err).ToNot(HaveOccurred()) - - riAfter, err := dir.AsResourceInfo(env.Ctx, &perms, []string{}, false) - Expect(err).ToNot(HaveOccurred()) - Expect(riAfter.Etag).ToNot(Equal(riBefore.Etag)) - }) - }) - - Describe("with TreeSizeAccounting enabled", func() { - It("calculates the size", func() { - file, err := env.CreateTestFile("file1", "", 1, dir.ID) - Expect(err).ToNot(HaveOccurred()) - - err = env.Tree.Propagate(env.Ctx, file) - Expect(err).ToNot(HaveOccurred()) - size, err := dir.GetTreeSize() - Expect(err).ToNot(HaveOccurred()) - Expect(size).To(Equal(uint64(1))) - }) - - It("considers all files", func() { - _, err := env.CreateTestFile("file1", "", 1, dir.ID) - Expect(err).ToNot(HaveOccurred()) - file2, err := env.CreateTestFile("file2", "", 100, dir.ID) - Expect(err).ToNot(HaveOccurred()) - - err = env.Tree.Propagate(env.Ctx, file2) - Expect(err).ToNot(HaveOccurred()) - size, err := dir.GetTreeSize() - Expect(err).ToNot(HaveOccurred()) - Expect(size).To(Equal(uint64(101))) - }) - - It("adds the size of child directories", func() { - subdir, err := env.CreateTestDir("testdir/200bytes") - Expect(err).ToNot(HaveOccurred()) - err = subdir.SetTreeSize(uint64(200)) - Expect(err).ToNot(HaveOccurred()) - - file, err := env.CreateTestFile("file1", "", 1, dir.ID) - Expect(err).ToNot(HaveOccurred()) - - err = env.Tree.Propagate(env.Ctx, file) - Expect(err).ToNot(HaveOccurred()) - size, err := dir.GetTreeSize() - Expect(err).ToNot(HaveOccurred()) - Expect(size).To(Equal(uint64(201))) - }) - - It("stops at nodes with no propagation flag", func() { - subdir, err := env.CreateTestDir("testdir/200bytes") - Expect(err).ToNot(HaveOccurred()) - err = subdir.SetTreeSize(uint64(200)) - Expect(err).ToNot(HaveOccurred()) - - err = env.Tree.Propagate(env.Ctx, subdir) - Expect(err).ToNot(HaveOccurred()) - size, err := dir.GetTreeSize() - Expect(size).To(Equal(uint64(200))) - Expect(err).ToNot(HaveOccurred()) - - stopdir, err := env.CreateTestDir("testdir/stophere") - Expect(err).ToNot(HaveOccurred()) - err = xattr.Set(stopdir.InternalPath(), xattrs.PropagationAttr, []byte("0")) - Expect(err).ToNot(HaveOccurred()) - otherdir, err := env.CreateTestDir("testdir/stophere/lotsofbytes") - Expect(err).ToNot(HaveOccurred()) - err = otherdir.SetTreeSize(uint64(100000)) - Expect(err).ToNot(HaveOccurred()) - err = env.Tree.Propagate(env.Ctx, otherdir) - Expect(err).ToNot(HaveOccurred()) - - size, err = dir.GetTreeSize() - Expect(err).ToNot(HaveOccurred()) - Expect(size).To(Equal(uint64(200))) - }) - }) - }) -}) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go deleted file mode 100644 index 50d5c4cb38..0000000000 --- a/pkg/storage/utils/decomposedfs/upload.go +++ /dev/null @@ -1,727 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "crypto/md5" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "fmt" - "hash" - "hash/adler32" - "io" - "os" - "path/filepath" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/storage/utils/chunking" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/utils" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/rs/zerolog" - tusd "github.com/tus/tusd/pkg/handler" -) - -var defaultFilePerm = os.FileMode(0664) - -// Upload uploads data to the given resource -// TODO Upload (and InitiateUpload) needs a way to receive the expected checksum. -// Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) (err error) { - upload, err := fs.GetUpload(ctx, ref.GetPath()) - if err != nil { - return errors.Wrap(err, "decomposedfs: error retrieving upload") - } - - uploadInfo := upload.(*fileUpload) - - p := uploadInfo.info.Storage["NodeName"] - ok, err := chunking.IsChunked(p) // check chunking v1 - if err != nil { - return errors.Wrap(err, "decomposedfs: error checking path") - } - if ok { - var assembledFile string - p, assembledFile, err = fs.chunkHandler.WriteChunk(p, r) - if err != nil { - return err - } - if p == "" { - if err = uploadInfo.Terminate(ctx); err != nil { - return errors.Wrap(err, "ocfs: error removing auxiliary files") - } - return errtypes.PartialContent(ref.String()) - } - uploadInfo.info.Storage["NodeName"] = p - fd, err := os.Open(assembledFile) - if err != nil { - return errors.Wrap(err, "decomposedfs: error opening assembled file") - } - defer fd.Close() - defer os.RemoveAll(assembledFile) - r = fd - } - - if _, err := uploadInfo.WriteChunk(ctx, 0, r); err != nil { - return errors.Wrap(err, "decomposedfs: error writing to binary file") - } - - return uploadInfo.FinishUpload(ctx) -} - -// InitiateUpload returns upload ids corresponding to different protocols it supports -// TODO read optional content for small files in this request -// TODO InitiateUpload (and Upload) needs a way to receive the expected checksum. Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - log := appctx.GetLogger(ctx) - - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, err - } - - // permissions are checked in NewUpload below - - relative, err := fs.lu.Path(ctx, n) - if err != nil { - return nil, err - } - - info := tusd.FileInfo{ - MetaData: tusd.MetaData{ - "filename": filepath.Base(relative), - "dir": filepath.Dir(relative), - }, - Size: uploadLength, - Storage: map[string]string{ - "SpaceRoot": n.SpaceRoot.ID, - }, - } - - if metadata != nil { - if metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] - } - if _, ok := metadata["sizedeferred"]; ok { - info.SizeIsDeferred = true - } - if metadata["checksum"] != "" { - parts := strings.SplitN(metadata["checksum"], " ", 2) - if len(parts) != 2 { - return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1", "md5", "adler32": - info.MetaData["checksum"] = metadata["checksum"] - default: - return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - } - } - - log.Debug().Interface("info", info).Interface("node", n).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename") - - _, err = node.CheckQuota(n.SpaceRoot, uint64(info.Size)) - if err != nil { - return nil, err - } - - upload, err := fs.NewUpload(ctx, info) - if err != nil { - return nil, err - } - - info, _ = upload.GetInfo(ctx) - - return map[string]string{ - "simple": info.ID, - "tus": info.ID, - }, nil -} - -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *Decomposedfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - composer.UseConcater(fs) - composer.UseLengthDeferrer(fs) -} - -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload - -// NewUpload returns a new tus Upload instance. -func (fs *Decomposedfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { - log := appctx.GetLogger(ctx) - log.Debug().Interface("info", info).Msg("Decomposedfs: NewUpload") - - fn := info.MetaData["filename"] - if fn == "" { - return nil, errors.New("Decomposedfs: missing filename in metadata") - } - info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) - - dir := info.MetaData["dir"] - if dir == "" { - return nil, errors.New("Decomposedfs: missing dir in metadata") - } - info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) - - n, err := fs.lookupNode(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: error wrapping filename") - } - - log.Debug().Interface("info", info).Interface("node", n).Msg("Decomposedfs: resolved filename") - - // the parent owner will become the new owner - p, perr := n.Parent() - if perr != nil { - return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID) - } - - // check permissions - var ok bool - if n.Exists { - // check permissions of file to be overwritten - ok, err = fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - return rp.InitiateFileUpload - }) - } else { - // check permissions of parent - ok, err = fs.p.HasPermission(ctx, p, func(rp *provider.ResourcePermissions) bool { - return rp.InitiateFileUpload - }) - } - switch { - case err != nil: - return nil, errtypes.InternalError(err.Error()) - case !ok: - return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - info.ID = uuid.New().String() - - binPath, err := fs.getUploadPath(ctx, info.ID) - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: error resolving upload path") - } - usr := ctxpkg.ContextMustGetUser(ctx) - - owner, err := p.Owner() - if err != nil { - return nil, errors.Wrap(err, "decomposedfs: error determining owner") - } - var spaceRoot string - if info.Storage != nil { - if spaceRoot, ok = info.Storage["SpaceRoot"]; !ok { - spaceRoot = n.SpaceRoot.ID - } - } else { - spaceRoot = n.SpaceRoot.ID - } - - info.Storage = map[string]string{ - "Type": "OCISStore", - "BinPath": binPath, - - "NodeId": n.ID, - "NodeParentId": n.ParentID, - "NodeName": n.Name, - "SpaceRoot": spaceRoot, - - "Idp": usr.Id.Idp, - "UserId": usr.Id.OpaqueId, - "UserType": utils.UserTypeToString(usr.Id.Type), - "UserName": usr.Username, - - "OwnerIdp": owner.Idp, - "OwnerId": owner.OpaqueId, - - "LogLevel": log.GetLevel().String(), - } - // Create binary file in the upload folder with no content - log.Debug().Interface("info", info).Msg("Decomposedfs: built storage info") - file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return nil, err - } - defer file.Close() - - u := &fileUpload{ - info: info, - binPath: binPath, - infoPath: filepath.Join(fs.o.Root, "uploads", info.ID+".info"), - fs: fs, - ctx: ctx, - } - - // writeInfo creates the file by itself if necessary - err = u.writeInfo() - if err != nil { - return nil, err - } - - return u, nil -} - -func (fs *Decomposedfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { - return filepath.Join(fs.o.Root, "uploads", uploadID), nil -} - -// GetUpload returns the Upload for the given upload id. -func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - infoPath := filepath.Join(fs.o.Root, "uploads", id+".info") - - info := tusd.FileInfo{} - data, err := os.ReadFile(infoPath) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - if err := json.Unmarshal(data, &info); err != nil { - return nil, err - } - - stat, err := os.Stat(info.Storage["BinPath"]) - if err != nil { - return nil, err - } - - info.Offset = stat.Size() - - u := &userpb.User{ - Id: &userpb.UserId{ - Idp: info.Storage["Idp"], - OpaqueId: info.Storage["UserId"], - Type: utils.UserTypeMap(info.Storage["UserType"]), - }, - Username: info.Storage["UserName"], - } - - ctx = ctxpkg.ContextSetUser(ctx, u) - // TODO configure the logger the same way ... store and add traceid in file info - - var opts []logger.Option - opts = append(opts, logger.WithLevel(info.Storage["LogLevel"])) - opts = append(opts, logger.WithWriter(os.Stderr, logger.ConsoleMode)) - l := logger.New(opts...) - - sub := l.With().Int("pid", os.Getpid()).Logger() - - ctx = appctx.WithLogger(ctx, &sub) - - return &fileUpload{ - info: info, - binPath: info.Storage["BinPath"], - infoPath: infoPath, - fs: fs, - ctx: ctx, - }, nil -} - -// lookupNode looks up nodes by path. -// This method can also handle lookups for paths which contain chunking information. -func (fs *Decomposedfs) lookupNode(ctx context.Context, path string) (*node.Node, error) { - p := path - isChunked, err := chunking.IsChunked(path) - if err != nil { - return nil, err - } - if isChunked { - chunkInfo, err := chunking.GetChunkBLOBInfo(path) - if err != nil { - return nil, err - } - p = chunkInfo.Path - } - - n, err := fs.lu.NodeFromPath(ctx, p, false) - if err != nil { - return nil, err - } - - if isChunked { - n.Name = filepath.Base(path) - } - return n, nil -} - -type fileUpload struct { - // info stores the current information about the upload - info tusd.FileInfo - // infoPath is the path to the .info file - infoPath string - // binPath is the path to the binary file (which has no extension) - binPath string - // only fs knows how to handle metadata and versions - fs *Decomposedfs - // a context with a user - // TODO add logger as well? - ctx context.Context -} - -// GetInfo returns the FileInfo. -func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { - return upload.info, nil -} - -// WriteChunk writes the stream from the reader to the given offset of the upload. -func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return 0, err - } - defer file.Close() - - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... - n, err := io.Copy(file, src) - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the ocis driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil { - if err != io.ErrUnexpectedEOF { - return n, err - } - } - - upload.info.Offset += n - err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk - - return n, err -} - -// GetReader returns an io.Reader for the upload. -func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { - return os.Open(upload.binPath) -} - -// writeInfo updates the entire information. Everything will be overwritten. -func (upload *fileUpload) writeInfo() error { - data, err := json.Marshal(upload.info) - if err != nil { - return err - } - return os.WriteFile(upload.infoPath, data, defaultFilePerm) -} - -// FinishUpload finishes an upload and moves the file to the internal destination. -func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { - // ensure cleanup - defer upload.discardChunk() - - fi, err := os.Stat(upload.binPath) - if err != nil { - appctx.GetLogger(upload.ctx).Err(err).Msg("Decomposedfs: could not stat uploaded file") - return - } - - n := node.New( - upload.info.Storage["NodeId"], - upload.info.Storage["NodeParentId"], - upload.info.Storage["NodeName"], - fi.Size(), - "", - nil, - upload.fs.lu, - ) - n.SpaceRoot = node.New(upload.info.Storage["SpaceRoot"], "", "", 0, "", nil, upload.fs.lu) - - _, err = node.CheckQuota(n.SpaceRoot, uint64(fi.Size())) - if err != nil { - return err - } - - if n.ID == "" { - n.ID = uuid.New().String() - } - targetPath := n.InternalPath() - sublog := appctx.GetLogger(upload.ctx). - With(). - Interface("info", upload.info). - Str("binPath", upload.binPath). - Str("targetPath", targetPath). - Logger() - - // calculate the checksum of the written bytes - // they will all be written to the metadata later, so we cannot omit any of them - // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present - // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - { - f, err := os.Open(upload.binPath) - if err != nil { - sublog.Err(err).Msg("Decomposedfs: could not open file for checksumming") - // we can continue if no oc checksum header is set - } - defer f.Close() - - r1 := io.TeeReader(f, sha1h) - r2 := io.TeeReader(r1, md5h) - - if _, err := io.Copy(adler32h, r2); err != nil { - sublog.Err(err).Msg("Decomposedfs: could not copy bytes for checksumming") - } - } - // compare if they match the sent checksum - // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - if upload.info.MetaData["checksum"] != "" { - parts := strings.SplitN(upload.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = upload.checkHash(parts[1], sha1h) - case "md5": - err = upload.checkHash(parts[1], md5h) - case "adler32": - err = upload.checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if err != nil { - return err - } - } - n.BlobID = upload.info.ID // This can be changed to a content hash in the future when reference counting for the blobs was added - - // defer writing the checksums until the node is in place - - // if target exists create new version - if fi, err = os.Stat(targetPath); err == nil { - // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - versionsPath := upload.fs.lu.InternalPath(n.ID + ".REV." + fi.ModTime().UTC().Format(time.RFC3339Nano)) - - if err = os.Rename(targetPath, versionsPath); err != nil { - sublog.Err(err). - Str("binPath", upload.binPath). - Str("versionsPath", versionsPath). - Msg("Decomposedfs: could not create version") - return - } - } - - // upload the data to the blobstore - file, err := os.Open(upload.binPath) - if err != nil { - return err - } - defer file.Close() - err = upload.fs.tp.WriteBlob(n.BlobID, file) - if err != nil { - return errors.Wrap(err, "failed to upload file to blostore") - } - - // now truncate the upload (the payload stays in the blobstore) and move it to the target path - // TODO put uploads on the same underlying storage as the destination dir? - // TODO trigger a workflow as the final rename might eg involve antivirus scanning - if err = os.Truncate(upload.binPath, 0); err != nil { - sublog.Err(err). - Msg("Decomposedfs: could not truncate") - return - } - if err = os.Rename(upload.binPath, targetPath); err != nil { - sublog.Err(err). - Msg("Decomposedfs: could not rename") - return - } - - // now try write all checksums - tryWritingChecksum(&sublog, n, "sha1", sha1h) - tryWritingChecksum(&sublog, n, "md5", md5h) - tryWritingChecksum(&sublog, n, "adler32", adler32h) - - // who will become the owner? the owner of the parent actually ... not the currently logged in user - err = n.WriteMetadata(&userpb.UserId{ - Idp: upload.info.Storage["OwnerIdp"], - OpaqueId: upload.info.Storage["OwnerId"], - }) - if err != nil { - return errors.Wrap(err, "decomposedfs: could not write metadata") - } - - // link child name to parent if it is new - childNameLink := filepath.Join(upload.fs.lu.InternalPath(n.ParentID), n.Name) - var link string - link, err = os.Readlink(childNameLink) - if err == nil && link != "../"+n.ID { - sublog.Err(err). - Interface("node", n). - Str("childNameLink", childNameLink). - Str("link", link). - Msg("Decomposedfs: child name link has wrong target id, repairing") - - if err = os.Remove(childNameLink); err != nil { - return errors.Wrap(err, "decomposedfs: could not remove symlink child entry") - } - } - if os.IsNotExist(err) || link != "../"+n.ID { - if err = os.Symlink("../"+n.ID, childNameLink); err != nil { - return errors.Wrap(err, "decomposedfs: could not symlink child entry") - } - } - - // only delete the upload if it was successfully written to the storage - if err = os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - sublog.Err(err).Msg("Decomposedfs: could not delete upload info") - return - } - } - // use set arbitrary metadata? - if upload.info.MetaData["mtime"] != "" { - err := n.SetMtime(ctx, upload.info.MetaData["mtime"]) - if err != nil { - sublog.Err(err).Interface("info", upload.info).Msg("Decomposedfs: could not set mtime metadata") - return err - } - } - - n.Exists = true - - return upload.fs.tp.Propagate(upload.ctx, n) -} - -func (upload *fileUpload) checkHash(expected string, h hash.Hash) error { - if expected != hex.EncodeToString(h.Sum(nil)) { - upload.discardChunk() - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", upload.info.MetaData["checksum"], h.Sum(nil))) - } - return nil -} -func tryWritingChecksum(log *zerolog.Logger, n *node.Node, algo string, h hash.Hash) { - if err := n.SetChecksum(algo, h); err != nil { - log.Err(err). - Str("csType", algo). - Bytes("hash", h.Sum(nil)). - Msg("Decomposedfs: could not write checksum") - // this is not critical, the bytes are there so we will continue - } -} - -func (upload *fileUpload) discardChunk() { - if err := os.Remove(upload.binPath); err != nil { - if !os.IsNotExist(err) { - appctx.GetLogger(upload.ctx).Err(err).Interface("info", upload.info).Str("binPath", upload.binPath).Interface("info", upload.info).Msg("Decomposedfs: could not discard chunk") - return - } - } - if err := os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - appctx.GetLogger(upload.ctx).Err(err).Interface("info", upload.info).Str("infoPath", upload.infoPath).Interface("info", upload.info).Msg("Decomposedfs: could not discard chunk info") - return - } - } -} - -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// - the storage needs to implement AsTerminatableUpload -// - the upload needs to implement Terminate - -// AsTerminatableUpload returns a TerminatableUpload. -func (fs *Decomposedfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { - return upload.(*fileUpload) -} - -// Terminate terminates the upload. -func (upload *fileUpload) Terminate(ctx context.Context) error { - if err := os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - return err - } - } - if err := os.Remove(upload.binPath); err != nil { - if !os.IsNotExist(err) { - return err - } - } - return nil -} - -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// - the storage needs to implement AsLengthDeclarableUpload -// - the upload needs to implement DeclareLength - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload. -func (fs *Decomposedfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { - return upload.(*fileUpload) -} - -// DeclareLength updates the upload length information. -func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { - upload.info.Size = length - upload.info.SizeIsDeferred = false - return upload.writeInfo() -} - -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// - the storage needs to implement AsConcatableUpload -// - the upload needs to implement ConcatUploads - -// AsConcatableUpload returns a ConcatableUpload. -func (fs *Decomposedfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { - return upload.(*fileUpload) -} - -// ConcatUploads concatenates multiple uploads. -func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*fileUpload) - - src, err := os.Open(fileUpload.binPath) - if err != nil { - return err - } - defer src.Close() - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - - return -} diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go deleted file mode 100644 index ae155c47ca..0000000000 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs_test - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ruser "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/mocks" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree" - treemocks "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree/mocks" - "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/xattr" - "github.com/stretchr/testify/mock" -) - -var _ = Describe("File uploads", func() { - var ( - ref *provider.Reference - fs storage.FS - user *userpb.User - ctx context.Context - - o *options.Options - lookup *decomposedfs.Lookup - permissions *mocks.PermissionsChecker - bs *treemocks.Blobstore - ) - - BeforeEach(func() { - ref = &provider.Reference{Path: "/foo"} - user = &userpb.User{ - Id: &userpb.UserId{ - Idp: "idp", - OpaqueId: "userid", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - Username: "username", - } - ctx = ruser.ContextSetUser(context.Background(), user) - - tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") - Expect(err).ToNot(HaveOccurred()) - - o, err = options.New(map[string]interface{}{ - "root": tmpRoot, - }) - Expect(err).ToNot(HaveOccurred()) - lookup = &decomposedfs.Lookup{Options: o} - permissions = &mocks.PermissionsChecker{} - bs = &treemocks.Blobstore{} - }) - - AfterEach(func() { - root := o.Root - if root != "" { - os.RemoveAll(root) - } - }) - - JustBeforeEach(func() { - var err error - tree := tree.New(o.Root, true, true, lookup, bs) - fs, err = decomposedfs.New(o, lookup, permissions, tree) - Expect(err).ToNot(HaveOccurred()) - }) - - Context("the user's quota is exceeded", func() { - When("the user wants to initiate a file upload", func() { - It("fails", func() { - var originalFunc = node.CheckQuota - node.CheckQuota = func(spaceRoot *node.Node, fileSize uint64) (quotaSufficient bool, err error) { - return false, errtypes.InsufficientStorage("quota exceeded") - } - _, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) - Expect(err).To(MatchError(errtypes.InsufficientStorage("quota exceeded"))) - node.CheckQuota = originalFunc - }) - }) - }) - - Context("the user has insufficient permissions", func() { - BeforeEach(func() { - permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - }) - - When("the user wants to initiate a file upload", func() { - It("fails", func() { - _, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) - Expect(err).To(MatchError("error: permission denied: root/foo")) - }) - }) - }) - - Context("with insufficient permissions, home node", func() { - BeforeEach(func() { - var err error - // recreate the fs with home enabled - o.EnableHome = true - tree := tree.New(o.Root, true, true, lookup, bs) - fs, err = decomposedfs.New(o, lookup, permissions, tree) - Expect(err).ToNot(HaveOccurred()) - err = fs.CreateHome(ctx) - Expect(err).ToNot(HaveOccurred()) - // the space name attribute is the stop condition in the lookup - h, err := lookup.HomeNode(ctx) - Expect(err).ToNot(HaveOccurred()) - err = xattr.Set(h.InternalPath(), xattrs.SpaceNameAttr, []byte("username")) - Expect(err).ToNot(HaveOccurred()) - permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) - }) - - When("the user wants to initiate a file upload", func() { - It("fails", func() { - h, err := lookup.HomeNode(ctx) - Expect(err).ToNot(HaveOccurred()) - msg := fmt.Sprintf("error: permission denied: %s/foo", h.ID) - _, err = fs.InitiateUpload(ctx, ref, 10, map[string]string{}) - Expect(err).To(MatchError(msg)) - }) - }) - }) - - Context("with sufficient permissions", func() { - BeforeEach(func() { - permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - permissions.On("AssemblePermissions", mock.Anything, mock.Anything). - Return(provider.ResourcePermissions{ - ListContainer: true, - }, nil) - }) - - When("the user initiates a non zero byte file upload", func() { - It("succeeds", func() { - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) - - rootRef := &provider.Reference{Path: "/"} - resources, err := fs.ListFolder(ctx, rootRef, []string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(resources)).To(Equal(0)) - }) - }) - - When("the user initiates a zero byte file upload", func() { - It("succeeds", func() { - uploadIds, err := fs.InitiateUpload(ctx, ref, 0, map[string]string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) - - rootRef := &provider.Reference{Path: "/"} - resources, err := fs.ListFolder(ctx, rootRef, []string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(resources)).To(Equal(0)) - }) - }) - - When("the user uploads a non zero byte file", func() { - It("succeeds", func() { - var ( - fileContent = []byte("0123456789") - ) - - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) - - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - bs.On("Upload", mock.AnythingOfType("string"), mock.AnythingOfType("*os.File")). - Return(nil). - Run(func(args mock.Arguments) { - reader := args.Get(1).(io.Reader) - data, err := io.ReadAll(reader) - - Expect(err).ToNot(HaveOccurred()) - Expect(data).To(Equal([]byte("0123456789"))) - }) - - err = fs.Upload(ctx, uploadRef, io.NopCloser(bytes.NewReader(fileContent))) - - Expect(err).ToNot(HaveOccurred()) - bs.AssertCalled(GinkgoT(), "Upload", mock.Anything, mock.Anything) - - rootRef := &provider.Reference{Path: "/"} - resources, err := fs.ListFolder(ctx, rootRef, []string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(resources)).To(Equal(1)) - Expect(resources[0].Path).To(Equal(ref.Path)) - }) - }) - - When("the user uploads a zero byte file", func() { - It("succeeds", func() { - var ( - fileContent = []byte("") - ) - - uploadIds, err := fs.InitiateUpload(ctx, ref, 0, map[string]string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) - - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - bs.On("Upload", mock.AnythingOfType("string"), mock.AnythingOfType("*os.File")). - Return(nil). - Run(func(args mock.Arguments) { - reader := args.Get(1).(io.Reader) - data, err := io.ReadAll(reader) - - Expect(err).ToNot(HaveOccurred()) - Expect(data).To(Equal([]byte(""))) - }) - - err = fs.Upload(ctx, uploadRef, io.NopCloser(bytes.NewReader(fileContent))) - - Expect(err).ToNot(HaveOccurred()) - bs.AssertCalled(GinkgoT(), "Upload", mock.Anything, mock.Anything) - - rootRef := &provider.Reference{Path: "/"} - resources, err := fs.ListFolder(ctx, rootRef, []string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(resources)).To(Equal(1)) - Expect(resources[0].Path).To(Equal(ref.Path)) - }) - }) - - When("the user tries to upload a file without intialising the upload", func() { - It("fails", func() { - var ( - fileContent = []byte("0123456789") - ) - - uploadRef := &provider.Reference{Path: "/some-non-existent-upload-reference"} - err := fs.Upload(ctx, uploadRef, io.NopCloser(bytes.NewReader(fileContent))) - - Expect(err).To(HaveOccurred()) - - rootRef := &provider.Reference{Path: "/"} - resources, err := fs.ListFolder(ctx, rootRef, []string{}) - - Expect(err).ToNot(HaveOccurred()) - Expect(len(resources)).To(Equal(0)) - }) - }) - - }) -}) diff --git a/pkg/storage/utils/decomposedfs/xattrs/xattrs.go b/pkg/storage/utils/decomposedfs/xattrs/xattrs.go deleted file mode 100644 index 388c2c7371..0000000000 --- a/pkg/storage/utils/decomposedfs/xattrs/xattrs.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package xattrs - -import ( - "strings" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" -) - -// Declare a list of xattr keys -// TODO the below comment is currently copied from the owncloud driver, revisit -// Currently,extended file attributes have four separated -// namespaces (user, trusted, security and system) followed by a dot. -// A non root user can only manipulate the user. namespace, which is what -// we will use to store ownCloud specific metadata. To prevent name -// collisions with other apps We are going to introduce a sub namespace -// "user.ocis.". -const ( - OcisPrefix string = "user.ocis." - ParentidAttr string = OcisPrefix + "parentid" - OwnerIDAttr string = OcisPrefix + "owner.id" - OwnerIDPAttr string = OcisPrefix + "owner.idp" - OwnerTypeAttr string = OcisPrefix + "owner.type" - // the base name of the node - // updated when the file is renamed or moved. - NameAttr string = OcisPrefix + "name" - - BlobIDAttr string = OcisPrefix + "blobid" - BlobsizeAttr string = OcisPrefix + "blobsize" - - // grantPrefix is the prefix for sharing related extended attributes. - GrantPrefix string = OcisPrefix + "grant." - MetadataPrefix string = OcisPrefix + "md." - - // favorite flag, per user. - FavPrefix string = OcisPrefix + "fav." - - // a temporary etag for a folder that is removed when the mtime propagation happens. - TmpEtagAttr string = OcisPrefix + "tmp.etag" - ReferenceAttr string = OcisPrefix + "cs3.ref" // arbitrary metadata - ChecksumPrefix string = OcisPrefix + "cs." // followed by the algorithm, eg. ocis.cs.sha1 - TrashOriginAttr string = OcisPrefix + "trash.origin" // trash origin - - // we use a single attribute to enable or disable propagation of both: synctime and treesize - // The propagation attribute is set to '1' at the top of the (sub)tree. Propagation will stop at - // that node. - PropagationAttr string = OcisPrefix + "propagation" - - // the tree modification time of the tree below this node, - // propagated when synctime_accounting is true and - // user.ocis.propagation=1 is set - // stored as a readable time.RFC3339Nano. - TreeMTimeAttr string = OcisPrefix + "tmtime" - - // the size of the tree below this node, - // propagated when treesize_accounting is true and - // user.ocis.propagation=1 is set - // stored as uint64, little endian. - TreesizeAttr string = OcisPrefix + "treesize" - - // the quota for the storage space / tree, regardless who accesses it. - QuotaAttr string = OcisPrefix + "quota" - - // the name given to a storage space. It should not contain any semantics as its only purpose is to be read. - SpaceNameAttr string = OcisPrefix + "space.name" - - UserAcePrefix string = "u:" - GroupAcePrefix string = "g:" -) - -// ReferenceFromAttr returns a CS3 reference from xattr of a node. -// Supported formats are: "cs3:storageid/nodeid". -func ReferenceFromAttr(b []byte) (*provider.Reference, error) { - return refFromCS3(b) -} - -// refFromCS3 creates a CS3 reference from a set of bytes. This method should remain private -// and only be called after validation because it can potentially panic. -func refFromCS3(b []byte) (*provider.Reference, error) { - parts := string(b[4:]) - return &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: strings.Split(parts, "/")[0], - OpaqueId: strings.Split(parts, "/")[1], - }, - }, nil -} diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go deleted file mode 100644 index 93aa760059..0000000000 --- a/pkg/trace/trace.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package trace - -import ( - "fmt" - "net/url" - "strings" - - "go.opentelemetry.io/otel/exporters/jaeger" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.4.0" - "go.opentelemetry.io/otel/trace" -) - -var ( - // Propagator is the default Reva propagator. - Propagator = propagation.NewCompositeTextMapPropagator(propagation.Baggage{}, propagation.TraceContext{}) - - // Provider is the default Reva tracer provider. - Provider = trace.NewNoopTracerProvider() -) - -// SetTraceProvider sets the TracerProvider at a package level. -func SetTraceProvider(collectorEndpoint string, agentEndpoint, serviceName string) { - // default to 'reva' as service name if not set - if serviceName == "" { - serviceName = "reva" - } - - var exp *jaeger.Exporter - var err error - - if agentEndpoint != "" { - var agentHost string - var agentPort string - - agentHost, agentPort, err = parseAgentConfig(agentEndpoint) - if err != nil { - panic(err) - } - - exp, err = jaeger.New( - jaeger.WithAgentEndpoint( - jaeger.WithAgentHost(agentHost), - jaeger.WithAgentPort(agentPort), - ), - ) - if err != nil { - panic(err) - } - } - - if collectorEndpoint != "" { - exp, err = jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collectorEndpoint))) - if err != nil { - panic(err) - } - } - - Provider = sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exp), - sdktrace.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(serviceName), - )), - ) -} - -func parseAgentConfig(ae string) (string, string, error) { - u, err := url.Parse(ae) - // as per url.go: - // [...] Trying to parse a hostname and path - // without a scheme is invalid but may not necessarily return an - // error, due to parsing ambiguities. - if err == nil && u.Hostname() != "" && u.Port() != "" { - return u.Hostname(), u.Port(), nil - } - - p := strings.Split(ae, ":") - if len(p) != 2 { - return "", "", fmt.Errorf(fmt.Sprintf("invalid agent endpoint `%s`. expected format: `hostname:port`", ae)) - } - - switch { - case p[0] == "" && p[1] == "": // case ae = ":" - return "", "", fmt.Errorf(fmt.Sprintf("invalid agent endpoint `%s`. expected format: `hostname:port`", ae)) - case p[0] == "": - return "", "", fmt.Errorf(fmt.Sprintf("invalid agent endpoint `%s`. expected format: `hostname:port`", ae)) - } - return p[0], p[1], nil -} From b132e3a1f993ee553c8b541825ceb510422783c2 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 6 Sep 2023 16:00:12 +0200 Subject: [PATCH 02/25] add helloworld example and fix auth logic to bail out early --- examples/helloworld/revad.toml | 4 ++++ internal/http/interceptors/auth/auth.go | 10 ++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 examples/helloworld/revad.toml diff --git a/examples/helloworld/revad.toml b/examples/helloworld/revad.toml new file mode 100644 index 0000000000..b9720c09d0 --- /dev/null +++ b/examples/helloworld/revad.toml @@ -0,0 +1,4 @@ +[http.services.helloworld] + +[http] +address = "localhost:8080" diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index c664c232f4..56d9023965 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -174,16 +174,14 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err if utils.Skip(r.URL.Path, unprotected) { log.Info().Msg("skipping auth check for: " + r.URL.Path) isUnprotectedEndpoint = true - } - - ctx, err := authenticateUser(w, r, conf, tokenStrategyChain, tokenManager, tokenWriter, credChain, isUnprotectedEndpoint) - if err != nil { - if !isUnprotectedEndpoint { + } else { + ctx, err := authenticateUser(w, r, conf, tokenStrategyChain, tokenManager, tokenWriter, credChain, isUnprotectedEndpoint) + if err != nil { return } - } else { r = r.WithContext(ctx) } + h.ServeHTTP(w, r) }) } From 7a38177a8f99eb2e433d99d680cace4ae94e4b79 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 6 Sep 2023 16:10:26 +0200 Subject: [PATCH 03/25] remove hack on appctx and add traceid --- .../grpc/services/storageprovider/storageprovider.go | 9 --------- pkg/appctx/appctx.go | 9 ++++++--- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 257f2b40f4..326b594d6a 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -713,15 +713,6 @@ func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*pro }, nil } - // check DeleteRequest for any known opaque properties. - if req.Opaque != nil { - _, ok := req.Opaque.Map["deleting_shared_resource"] - if ok { - // it is a binary key; its existence signals true. Although, do not assume. - ctx = context.WithValue(ctx, appctx.DeletingSharedResource, true) - } - } - if err := s.storage.Delete(ctx, newRef); err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/appctx/appctx.go b/pkg/appctx/appctx.go index 3aad813443..67ac9f13dc 100644 --- a/pkg/appctx/appctx.go +++ b/pkg/appctx/appctx.go @@ -22,11 +22,9 @@ import ( "context" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" ) -// DeletingSharedResource flags to a storage a shared resource is being deleted not by the owner. -var DeletingSharedResource struct{} - // WithLogger returns a context with an associated logger. func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { return l.WithContext(ctx) @@ -37,3 +35,8 @@ func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { func GetLogger(ctx context.Context) *zerolog.Logger { return zerolog.Ctx(ctx) } + +func GetTraceID(ctx context.Context) trace.TraceID { + traceID := trace.SpanContextFromContext(ctx).TraceID() + return traceID +} From 8a2831e76a13cb30f52edc7446a613ed993412ee Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 12 Sep 2023 16:42:04 +0200 Subject: [PATCH 04/25] configurable prom collectors --- cmd/revad/runtime/http.go | 2 + cmd/revad/runtime/loader.go | 1 + examples/ceph/ceph.conf | 3 - examples/ceph/keyring | 2 - examples/helloworld/revad.toml | 1 + internal/http/interceptors/metrics/metrics.go | 90 +++++++++++++++++++ .../http/services/prometheus/prometheus.go | 33 ++++--- pkg/appctx/appctx.go | 6 +- pkg/prom/base/base.go | 38 ++++++++ pkg/prom/loader/loader.go | 26 ++++++ pkg/prom/registry/registry.go | 19 ++++ pkg/rhttp/rhttp.go | 12 +++ 12 files changed, 217 insertions(+), 16 deletions(-) delete mode 100644 examples/ceph/ceph.conf delete mode 100644 examples/ceph/keyring create mode 100644 internal/http/interceptors/metrics/metrics.go create mode 100644 pkg/prom/base/base.go create mode 100644 pkg/prom/loader/loader.go create mode 100644 pkg/prom/registry/registry.go diff --git a/cmd/revad/runtime/http.go b/cmd/revad/runtime/http.go index 7568c174ea..3af6d00369 100644 --- a/cmd/revad/runtime/http.go +++ b/cmd/revad/runtime/http.go @@ -25,6 +25,7 @@ import ( "github.com/cs3org/reva/internal/http/interceptors/appctx" "github.com/cs3org/reva/internal/http/interceptors/auth" "github.com/cs3org/reva/internal/http/interceptors/log" + "github.com/cs3org/reva/internal/http/interceptors/metrics" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -67,6 +68,7 @@ func initHTTPMiddlewares(conf map[string]map[string]any, unprotected []string, l } middlewares := []global.Middleware{ + metrics.New(), authMiddle, log.New(), appctx.New(*logger), diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index ad174b9130..aae2d2d689 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -44,6 +44,7 @@ import ( _ "github.com/cs3org/reva/pkg/ocm/share/repository/loader" _ "github.com/cs3org/reva/pkg/permission/manager/loader" _ "github.com/cs3org/reva/pkg/preferences/loader" + _ "github.com/cs3org/reva/pkg/prom/loader" _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/rhttp/datatx/manager/loader" _ "github.com/cs3org/reva/pkg/share/cache/loader" diff --git a/examples/ceph/ceph.conf b/examples/ceph/ceph.conf deleted file mode 100644 index f2daaa8de5..0000000000 --- a/examples/ceph/ceph.conf +++ /dev/null @@ -1,3 +0,0 @@ -[global] - fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323' - mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]' diff --git a/examples/ceph/keyring b/examples/ceph/keyring deleted file mode 100644 index 9e555cc1b2..0000000000 --- a/examples/ceph/keyring +++ /dev/null @@ -1,2 +0,0 @@ -[client.admin] - key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA==' diff --git a/examples/helloworld/revad.toml b/examples/helloworld/revad.toml index b9720c09d0..b5b45b1390 100644 --- a/examples/helloworld/revad.toml +++ b/examples/helloworld/revad.toml @@ -1,4 +1,5 @@ [http.services.helloworld] +[http.services.prometheus] [http] address = "localhost:8080" diff --git a/internal/http/interceptors/metrics/metrics.go b/internal/http/interceptors/metrics/metrics.go new file mode 100644 index 0000000000..c1635fda5c --- /dev/null +++ b/internal/http/interceptors/metrics/metrics.go @@ -0,0 +1,90 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package appctx creates a context with useful +// components attached to the context like loggers and +// token managers. +package metrics + +import ( + "context" + "github.com/cs3org/reva/pkg/prom/registry" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" +) + +var inFlightGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http_in_flight_requests", + Help: "A gauge of requests currently being served by the wrapped handler.", +}) + +var counter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_api_requests_total", + Help: "A counter for requests to the wrapped handler.", + }, + []string{"code", "method"}, +) + +// duration is partitioned by the HTTP method and handler. It uses custom +// buckets based on the expected request duration. +var duration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "A histogram of latencies for requests.", + Buckets: []float64{.25, .5, 1, 2.5, 5, 10}, + }, + []string{"handler", "method"}, +) + +// responseSize has no labels, making it a zero-dimensional +// ObserverVec. +var responseSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_response_size_bytes", + Help: "A histogram of response sizes for requests.", + Buckets: []float64{200, 500, 900, 1500}, + }, + []string{}, +) + +func init() { + registry.Register("http_metrics", NewPromCollectors) +} + +// New returns a prometheus collector +func NewPromCollectors(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { + return []prometheus.Collector{inFlightGauge, counter, duration, responseSize}, nil +} + +// New returns a new HTTP middleware that stores the log +// in the context with request ID information. +func New() func(h http.Handler) http.Handler { + chain := func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h = promhttp.InstrumentHandlerDuration(duration.MustCurryWith(prometheus.Labels{"handler": r.URL.Path}), + promhttp.InstrumentHandlerCounter(counter, + promhttp.InstrumentHandlerResponseSize(responseSize, h), + ), + ) + h.ServeHTTP(w, r) + }) + } + return chain +} diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index b458bffbd4..35b235eaec 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -22,11 +22,11 @@ import ( "context" "net/http" - "contrib.go.opencensus.io/exporter/prometheus" + "github.com/cs3org/reva/pkg/prom/registry" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/utils/cfg" - "github.com/pkg/errors" - "go.opencensus.io/stats/view" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) func init() { @@ -40,15 +40,28 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) return nil, err } - pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: "revad", - }) - if err != nil { - return nil, errors.Wrap(err, "prometheus: error creating exporter") + // instantiate and register all collectors + collectors := []prometheus.Collector{} + for _, f := range registry.NewFuncs { + cols, err := f(ctx, m) + if err != nil { + return nil, err + } + collectors = append(collectors, cols...) } - view.RegisterExporter(pe) - return &svc{prefix: c.Prefix, h: pe}, nil + // custom registry to avoid global prometheus registry that can be + // modified at global package level + reg := prometheus.NewRegistry() + reg.MustRegister(collectors...) + + handler := promhttp.HandlerFor( + reg, + promhttp.HandlerOpts{ + Registry: reg, + EnableOpenMetrics: true, + }) + return &svc{prefix: c.Prefix, h: handler}, nil } type config struct { diff --git a/pkg/appctx/appctx.go b/pkg/appctx/appctx.go index 67ac9f13dc..683f48beb7 100644 --- a/pkg/appctx/appctx.go +++ b/pkg/appctx/appctx.go @@ -25,9 +25,13 @@ import ( "go.opentelemetry.io/otel/trace" ) +const traceIDKey = "traceid" + // WithLogger returns a context with an associated logger. func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { - return l.WithContext(ctx) + traceID := GetTraceID(ctx) + sublog := l.With().Str(traceIDKey, traceID.String()).Logger() + return sublog.WithContext(ctx) } // GetLogger returns the logger associated with the given context diff --git a/pkg/prom/base/base.go b/pkg/prom/base/base.go new file mode 100644 index 0000000000..6f598d03ac --- /dev/null +++ b/pkg/prom/base/base.go @@ -0,0 +1,38 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package base + +import ( + "context" + "github.com/cs3org/reva/pkg/prom/registry" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +func init() { + registry.Register("base", New) +} + +// New returns a prometheus collector +func New(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { + return []prometheus.Collector{ + collectors.NewBuildInfoCollector(), + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})}, nil +} diff --git a/pkg/prom/loader/loader.go b/pkg/prom/loader/loader.go new file mode 100644 index 0000000000..35daf099a7 --- /dev/null +++ b/pkg/prom/loader/loader.go @@ -0,0 +1,26 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load collectors + _ "github.com/cs3org/reva/internal/http/interceptors/metrics" + _ "github.com/cs3org/reva/pkg/prom/base" + // Add your own here. +) diff --git a/pkg/prom/registry/registry.go b/pkg/prom/registry/registry.go new file mode 100644 index 0000000000..c2f60c6555 --- /dev/null +++ b/pkg/prom/registry/registry.go @@ -0,0 +1,19 @@ +package registry + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" +) + +// NewFunc is the function that custom prometheus collectors implement +// should register at init time. +type NewFunc func(context.Context, map[string]interface{}) ([]prometheus.Collector, error) + +// NewFuncs is a map containing all the registered collectors +var NewFuncs = map[string]NewFunc{} + +// Register registers a new prometheus collector new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index c4aafe9c07..b516814a6e 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -279,3 +279,15 @@ func (s *Server) getHandler() (http.Handler, error) { return handler, nil } + +// prometheusMiddleware implements mux.MiddlewareFunc. +/* +func prometheusHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + timer := prometheus.NewTimer(httpDuration.WithLabelValues(path)) + next.ServeHTTP(w, r) + timer.ObserveDuration() + }) +} +*/ From bc289b02241c9df1c1edfb373547bf89a291c71a Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 13 Sep 2023 12:01:34 +0200 Subject: [PATCH 05/25] grpc reflection working --- .gitignore | 1 + cmd/revad/runtime/grpc.go | 13 +- cmd/revad/runtime/runtime.go | 2 +- examples/helloworld/revad.toml | 5 + go.mod | 28 +- go.sum | 55 +- internal/grpc/interceptors/metrics/metrics.go | 51 ++ .../grpc/services/helloworld/helloworld.go | 2 +- internal/http/interceptors/metrics/metrics.go | 17 +- pkg/prom/loader/loader.go | 1 + .../integration/grpc/storageprovider_test.go | 650 ------------------ 11 files changed, 99 insertions(+), 726 deletions(-) create mode 100644 internal/grpc/interceptors/metrics/metrics.go delete mode 100644 tests/integration/grpc/storageprovider_test.go diff --git a/.gitignore b/.gitignore index 119d753fd0..45b92b2ad3 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ cmd/revad/revad cmd/revad/revad.pid cmd/reva/reva +cmd/revad/main/main tools/release/release # Ignore pid files diff --git a/cmd/revad/runtime/grpc.go b/cmd/revad/runtime/grpc.go index 35a70f8a24..a41309cfa4 100644 --- a/cmd/revad/runtime/grpc.go +++ b/cmd/revad/runtime/grpc.go @@ -24,6 +24,7 @@ import ( "github.com/cs3org/reva/internal/grpc/interceptors/appctx" "github.com/cs3org/reva/internal/grpc/interceptors/auth" "github.com/cs3org/reva/internal/grpc/interceptors/log" + "github.com/cs3org/reva/internal/grpc/interceptors/metrics" "github.com/cs3org/reva/internal/grpc/interceptors/recovery" "github.com/cs3org/reva/internal/grpc/interceptors/token" "github.com/cs3org/reva/internal/grpc/interceptors/useragent" @@ -80,6 +81,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, } unaryInterceptors = append([]grpc.UnaryServerInterceptor{ + metrics.NewUnary(), appctx.NewUnary(*logger), token.NewUnary(), useragent.NewUnary(), @@ -123,7 +125,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, } streamInterceptors = append([]grpc.StreamServerInterceptor{ - authStream, + metrics.NewStream(), appctx.NewStream(*logger), token.NewStream(), useragent.NewStream(), @@ -134,7 +136,14 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, return unaryInterceptors, streamInterceptors, nil } -func grpcUnprotected(s map[string]rgrpc.Service) (unprotected []string) { +func grpcUnprotected(reflection bool, s map[string]rgrpc.Service) (unprotected []string) { + if reflection { + // TODO(labkode): do not hardcode service endpoint and try to obtain from reflection library + unprotected = append(unprotected, + "/grpc.reflection.v1alpha.ServerReflection", + "/grpc.reflection.v1.ServerReflection", + ) + } for _, svc := range s { unprotected = append(unprotected, svc.UnprotectedEndpoints()...) } diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index d3627d8b80..44fa47a530 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -410,7 +410,7 @@ func newServers(ctx context.Context, grpc []*config.GRPC, http []*config.HTTP, l if err != nil { return nil, err } - unaryChain, streamChain, err := initGRPCInterceptors(cfg.Interceptors, grpcUnprotected(services), log) + unaryChain, streamChain, err := initGRPCInterceptors(cfg.Interceptors, grpcUnprotected(cfg.EnableReflection, services), log) if err != nil { return nil, err } diff --git a/examples/helloworld/revad.toml b/examples/helloworld/revad.toml index b5b45b1390..f567c8931a 100644 --- a/examples/helloworld/revad.toml +++ b/examples/helloworld/revad.toml @@ -1,5 +1,10 @@ [http.services.helloworld] +[grpc.services.helloworld] [http.services.prometheus] [http] address = "localhost:8080" + +[grpc] +address = "localhost:8081" +enable_reflection = true diff --git a/go.mod b/go.mod index e5abb46e80..fa23c57abe 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,10 @@ module github.com/cs3org/reva require ( bou.ke/monkey v1.0.2 - contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/BurntSushi/toml v1.2.1 github.com/Masterminds/sprig v2.22.0+incompatible github.com/ReneKroon/ttlcache/v2 v2.11.0 github.com/asim/go-micro/plugins/events/nats/v4 v4.0.0-20220118152736-9e0be6c85d75 - github.com/aws/aws-sdk-go v1.45.1 github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 @@ -35,12 +33,12 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v1.14.17 github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.2.1 - github.com/minio/minio-go/v7 v7.0.63 github.com/mitchellh/mapstructure v1.5.0 github.com/nats-io/nats-server/v2 v2.9.19 github.com/nats-io/nats-streaming-server v0.25.5 @@ -50,6 +48,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.5 github.com/prometheus/alertmanager v0.25.1 + github.com/prometheus/client_golang v1.16.0 github.com/rs/cors v1.8.3 github.com/rs/zerolog v1.28.0 github.com/sethvargo/go-password v0.2.0 @@ -60,10 +59,6 @@ require ( github.com/wk8/go-ordered-map v1.0.0 go-micro.dev/v4 v4.3.1-0.20211108085239-0c2041e43908 go.opencensus.io v0.24.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 - go.opentelemetry.io/otel v1.11.2 - go.opentelemetry.io/otel/exporters/jaeger v1.11.2 - go.opentelemetry.io/otel/sdk v1.11.2 go.opentelemetry.io/otel/trace v1.11.2 go.step.sm/crypto v0.35.0 golang.org/x/crypto v0.12.0 @@ -96,10 +91,6 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-kit/kit v0.10.0 // indirect - github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect github.com/gocraft/dbr/v2 v2.7.2 // indirect @@ -107,6 +98,7 @@ require ( github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect @@ -115,10 +107,7 @@ require ( github.com/hashicorp/raft v1.5.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lestrrat-go/strftime v1.0.4 // indirect github.com/mattn/go-colorable v0.1.12 // indirect @@ -128,13 +117,9 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.43 // indirect github.com/minio/highwayhash v1.0.2 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nats-io/jwt/v2 v2.4.1 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect @@ -146,12 +131,9 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/term v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.11.0 // indirect - github.com/prometheus/statsd_exporter v0.22.7 // indirect - github.com/rs/xid v1.5.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect @@ -160,17 +142,15 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.mongodb.org/mongo-driver v1.11.0 // indirect - go.opentelemetry.io/otel/metric v0.34.0 // indirect + go.opentelemetry.io/otel v1.11.2 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e8b0707d15..65ee0f384e 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= @@ -213,12 +212,10 @@ cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= @@ -760,8 +757,6 @@ cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= -contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -865,7 +860,6 @@ github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.37.27/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.45.1 h1:PXuxDZIo/Y9Bvtg2t055+dY4hRwNAEcq6bUMv9fXcjk= github.com/aws/aws-sdk-go v1.45.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= @@ -929,7 +923,6 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -1019,7 +1012,6 @@ github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exoscale/egoscale v0.46.0/go.mod h1:mpEXBpROAa/2i5GC0r33rfxG+TxSEka11g1PIXt9+zc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -1074,7 +1066,6 @@ github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= @@ -1083,13 +1074,8 @@ github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXg github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= @@ -1280,6 +1266,10 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5YbLR0/n2+wBxUpgf3HgfKgTyLI= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -1350,9 +1340,7 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4Fw github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= @@ -1365,7 +1353,6 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -1388,10 +1375,7 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1485,12 +1469,6 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -1514,11 +1492,9 @@ github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7s github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -1641,8 +1617,6 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= @@ -1664,7 +1638,6 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= @@ -1682,8 +1655,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= -github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1698,8 +1669,6 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1784,7 +1753,6 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 h1:VsBj3UD2xyAOu7kJw6O/2jjG2UXLFoBzihqDU9Ofg9M= github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= -github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/thanhpk/randstr v1.0.4 h1:IN78qu/bR+My+gHCvMEXhR/i5oriVHcTB/BJJIRTsNo= @@ -1857,16 +1825,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= -go.opentelemetry.io/otel/exporters/jaeger v1.11.2 h1:ES8/j2+aB+3/BUw51ioxa50V9btN1eew/2J7N7n1tsE= -go.opentelemetry.io/otel/exporters/jaeger v1.11.2/go.mod h1:nwcF/DK4Hk0auZ/a5vw20uMsaJSXbzeeimhN5f9d0Lc= -go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= -go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= -go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= -go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -2031,6 +1991,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -2073,6 +2034,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -2231,7 +2193,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2501,6 +2462,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2688,6 +2650,7 @@ google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2727,8 +2690,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ns1/ns1-go.v2 v2.4.4/go.mod h1:GMnKY+ZuoJ+lVLL+78uSTjwTz2jMazq6AfGKQOYhsPk= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/internal/grpc/interceptors/metrics/metrics.go b/internal/grpc/interceptors/metrics/metrics.go new file mode 100644 index 0000000000..899112ef64 --- /dev/null +++ b/internal/grpc/interceptors/metrics/metrics.go @@ -0,0 +1,51 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package metrics + +import ( + "context" + + "github.com/cs3org/reva/pkg/prom/registry" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" +) + +var collector = grpcprom.NewServerMetrics() + +func init() { + registry.Register("grpc_metrics", NewPromCollectors) +} + +// New returns a prometheus collector +func NewPromCollectors(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { + return []prometheus.Collector{collector}, nil +} + +// NewUnary returns a new unary interceptor that adds +// the useragent to the context. +func NewUnary() grpc.UnaryServerInterceptor { + return collector.UnaryServerInterceptor() +} + +// NewStream returns a new server stream interceptor +// that adds the user agent to the context. +func NewStream() grpc.StreamServerInterceptor { + return collector.StreamServerInterceptor() +} diff --git a/internal/grpc/services/helloworld/helloworld.go b/internal/grpc/services/helloworld/helloworld.go index 8d4eab0566..db7bab0fa2 100644 --- a/internal/grpc/services/helloworld/helloworld.go +++ b/internal/grpc/services/helloworld/helloworld.go @@ -62,7 +62,7 @@ func (s *service) Close() error { } func (s *service) UnprotectedEndpoints() []string { - return []string{} + return []string{"/revad.helloworld.HelloWorldService/Hello"} } func (s *service) Register(ss *grpc.Server) { diff --git a/internal/http/interceptors/metrics/metrics.go b/internal/http/interceptors/metrics/metrics.go index c1635fda5c..1311ccacb9 100644 --- a/internal/http/interceptors/metrics/metrics.go +++ b/internal/http/interceptors/metrics/metrics.go @@ -64,6 +64,17 @@ var responseSize = prometheus.NewHistogramVec( []string{}, ) +// requestSize has no labels, making it a zero-dimensional +// ObserverVec. +var requestSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_size_bytes", + Help: "A histogram of request sizes for requests.", + Buckets: []float64{200, 500, 900, 1500}, + }, + []string{}, +) + func init() { registry.Register("http_metrics", NewPromCollectors) } @@ -80,7 +91,11 @@ func New() func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h = promhttp.InstrumentHandlerDuration(duration.MustCurryWith(prometheus.Labels{"handler": r.URL.Path}), promhttp.InstrumentHandlerCounter(counter, - promhttp.InstrumentHandlerResponseSize(responseSize, h), + promhttp.InstrumentHandlerResponseSize(responseSize, + promhttp.InstrumentHandlerRequestSize(requestSize, + promhttp.InstrumentHandlerInFlight(inFlightGauge, h), + ), + ), ), ) h.ServeHTTP(w, r) diff --git a/pkg/prom/loader/loader.go b/pkg/prom/loader/loader.go index 35daf099a7..f62329d8ee 100644 --- a/pkg/prom/loader/loader.go +++ b/pkg/prom/loader/loader.go @@ -20,6 +20,7 @@ package loader import ( // Load collectors + _ "github.com/cs3org/reva/internal/grpc/interceptors/metrics" _ "github.com/cs3org/reva/internal/http/interceptors/metrics" _ "github.com/cs3org/reva/pkg/prom/base" // Add your own here. diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go deleted file mode 100644 index 30f2460244..0000000000 --- a/tests/integration/grpc/storageprovider_test.go +++ /dev/null @@ -1,650 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package grpc_test - -import ( - "context" - "os" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - storagep "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/storage/fs/ocis" - "github.com/cs3org/reva/pkg/storage/fs/owncloud" - jwt "github.com/cs3org/reva/pkg/token/manager/jwt" - "github.com/cs3org/reva/tests/helpers" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "google.golang.org/grpc/metadata" -) - -// This test suite tests the gprc storageprovider interface using different -// storage backends -// -// It uses the `startRevads` helper to spawn the according reva daemon and -// other dependencies like a userprovider if needed. -// It also sets up an authenticated context and a service client to the storage -// provider to be used in the assertion functions. -var _ = Describe("storage providers", func() { - var ( - dependencies = map[string]string{} - variables = map[string]string{} - revads = map[string]*Revad{} - - ctx context.Context - serviceClient storagep.ProviderAPIClient - user = &userpb.User{ - Id: &userpb.UserId{ - Idp: "0.0.0.0:19000", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - Username: "einstein", - } - - homeRef = &storagep.Reference{Path: "/"} - filePath = "/file" - fileRef = &storagep.Reference{Path: filePath} - versionedFilePath = "/versionedFile" - versionedFileRef = &storagep.Reference{Path: versionedFilePath} - subdirPath = "/subdir" - subdirRef = &storagep.Reference{Path: subdirPath} - sharesPath = "/Shares" - sharesRef = &storagep.Reference{Path: sharesPath} - ) - - JustBeforeEach(func() { - var err error - ctx = context.Background() - - // Add auth token - tokenManager, err := jwt.New(map[string]interface{}{"secret": "changemeplease"}) - Expect(err).ToNot(HaveOccurred()) - scope, err := scope.AddOwnerScope(nil) - Expect(err).ToNot(HaveOccurred()) - t, err := tokenManager.MintToken(ctx, user, scope) - Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) - - revads, err = startRevads(dependencies, nil, nil, variables) - Expect(err).ToNot(HaveOccurred()) - serviceClient, err = pool.GetStorageProviderServiceClient(pool.Endpoint(revads["storage"].GrpcAddress)) - Expect(err).ToNot(HaveOccurred()) - }) - - AfterEach(func() { - for _, r := range revads { - Expect(r.Cleanup(CurrentGinkgoTestDescription().Failed)).To(Succeed()) - } - }) - - assertCreateHome := func() { - It("creates a home directory", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - - res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(err).ToNot(HaveOccurred()) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - ghRes, err := serviceClient.GetHome(ctx, &storagep.GetHomeRequest{}) - Expect(err).ToNot(HaveOccurred()) - Expect(ghRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - } - - assertCreateContainer := func() { - It("creates a new directory", func() { - newRef := &storagep.Reference{Path: "/newdir"} - - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: newRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - - res, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: newRef}) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(err).ToNot(HaveOccurred()) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: newRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - } - - assertListContainer := func() { - It("lists a directory", func() { - listRes, err := serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(listRes.Infos)).To(Equal(1)) - - info := listRes.Infos[0] - Expect(info.Type).To(Equal(storagep.ResourceType_RESOURCE_TYPE_CONTAINER)) - Expect(info.Path).To(Equal(subdirPath)) - Expect(info.Owner.OpaqueId).To(Equal("f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c")) - }) - } - - assertFileVersions := func() { - It("lists file versions", func() { - listRes, err := serviceClient.ListFileVersions(ctx, &storagep.ListFileVersionsRequest{Ref: versionedFileRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(listRes.Versions)).To(Equal(1)) - Expect(listRes.Versions[0].Size).To(Equal(uint64(1))) - }) - - It("restores a file version", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: versionedFileRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(statRes.Info.Size).To(Equal(uint64(2))) // second version contains 2 bytes - - listRes, err := serviceClient.ListFileVersions(ctx, &storagep.ListFileVersionsRequest{Ref: versionedFileRef}) - Expect(err).ToNot(HaveOccurred()) - restoreRes, err := serviceClient.RestoreFileVersion(ctx, - &storagep.RestoreFileVersionRequest{ - Ref: versionedFileRef, - Key: listRes.Versions[0].Key, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(restoreRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: versionedFileRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(statRes.Info.Size).To(Equal(uint64(1))) // initial version contains 1 byte - }) - } - - assertDelete := func() { - It("deletes a directory", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(err).ToNot(HaveOccurred()) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - }) - } - - assertMove := func() { - It("moves a directory", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - targetRef := &storagep.Reference{Path: "/new_subdir"} - res, err := serviceClient.Move(ctx, &storagep.MoveRequest{Source: subdirRef, Destination: targetRef}) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(err).ToNot(HaveOccurred()) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: targetRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - } - - assertGetPath := func() { - It("gets the path to an ID", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - - res, err := serviceClient.GetPath(ctx, &storagep.GetPathRequest{ResourceId: statRes.Info.Id}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Path).To(Equal(subdirPath)) - }) - } - - assertGrants := func() { - It("lists, adds and removes grants", func() { - By("there are no grants initially") - listRes, err := serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(listRes.Grants)).To(Equal(0)) - - By("adding a grant") - grant := &storagep.Grant{ - Grantee: &storagep.Grantee{ - Type: storagep.GranteeType_GRANTEE_TYPE_USER, - Id: &storagep.Grantee_UserId{ - UserId: &userpb.UserId{ - OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51", - }, - }, - }, - Permissions: &storagep.ResourcePermissions{ - Stat: true, - Move: true, - Delete: false, - }, - } - addRes, err := serviceClient.AddGrant(ctx, &storagep.AddGrantRequest{Ref: subdirRef, Grant: grant}) - Expect(err).ToNot(HaveOccurred()) - Expect(addRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("listing the new grant") - listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(listRes.Grants)).To(Equal(1)) - readGrant := listRes.Grants[0] - Expect(readGrant.Permissions.Stat).To(BeTrue()) - Expect(readGrant.Permissions.Move).To(BeTrue()) - Expect(readGrant.Permissions.Delete).To(BeFalse()) - - By("updating the grant") - grant.Permissions.Delete = true - updateRes, err := serviceClient.UpdateGrant(ctx, &storagep.UpdateGrantRequest{Ref: subdirRef, Grant: grant}) - Expect(err).ToNot(HaveOccurred()) - Expect(updateRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("listing the update grant") - listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(listRes.Grants)).To(Equal(1)) - readGrant = listRes.Grants[0] - Expect(readGrant.Permissions.Stat).To(BeTrue()) - Expect(readGrant.Permissions.Move).To(BeTrue()) - Expect(readGrant.Permissions.Delete).To(BeTrue()) - - By("deleting a grant") - delRes, err := serviceClient.RemoveGrant(ctx, &storagep.RemoveGrantRequest{Ref: subdirRef, Grant: readGrant}) - Expect(err).ToNot(HaveOccurred()) - Expect(delRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("the grant is gone") - listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(listRes.Grants)).To(Equal(0)) - }) - } - - assertUploads := func() { - It("returns upload URLs for simple and tus", func() { - res, err := serviceClient.InitiateFileUpload(ctx, &storagep.InitiateFileUploadRequest{Ref: fileRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(res.Protocols)).To(Equal(2)) - }) - } - - assertDownloads := func() { - It("returns 'simple' download URLs", func() { - res, err := serviceClient.InitiateFileDownload(ctx, &storagep.InitiateFileDownloadRequest{Ref: fileRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(res.Protocols)).To(Equal(1)) - }) - } - - assertRecycle := func() { - It("lists and restores resources", func() { - By("deleting an item") - res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("listing the recycle items") - listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - Expect(len(listRes.RecycleItems)).To(Equal(1)) - item := listRes.RecycleItems[0] - Expect(item.Ref.Path).To(Equal(subdirPath)) - - By("restoring a recycle item") - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - - restoreRes, err := serviceClient.RestoreRecycleItem(ctx, - &storagep.RestoreRecycleItemRequest{ - Ref: homeRef, - Key: item.Key, - }, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(restoreRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - - It("restores resources to a different location", func() { - restoreRef := &storagep.Reference{Path: "/subdirRestored"} - By("deleting an item") - res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("listing the recycle items") - listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - Expect(len(listRes.RecycleItems)).To(Equal(1)) - item := listRes.RecycleItems[0] - Expect(item.Ref.Path).To(Equal(subdirPath)) - - By("restoring the item to a different location") - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: restoreRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - - restoreRes, err := serviceClient.RestoreRecycleItem(ctx, - &storagep.RestoreRecycleItemRequest{ - Ref: homeRef, - Key: item.Key, - RestoreRef: &storagep.Reference{Path: "/subdirRestored"}, - }, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(restoreRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: restoreRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - - It("purges recycle items resources", func() { - By("deleting an item") - res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - By("listing recycle items") - listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(listRes.RecycleItems)).To(Equal(1)) - - By("purging a recycle item") - purgeRes, err := serviceClient.PurgeRecycle(ctx, &storagep.PurgeRecycleRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(purgeRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - listRes, err = serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{Ref: homeRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(listRes.RecycleItems)).To(Equal(0)) - }) - } - - assertReferences := func() { - It("creates references", func() { - listRes, err := serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: sharesRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - Expect(len(listRes.Infos)).To(Equal(0)) - - res, err := serviceClient.CreateReference(ctx, &storagep.CreateReferenceRequest{ - Ref: &storagep.Reference{Path: "/Shares/reference"}, - TargetUri: "scheme://target", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - listRes, err = serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: sharesRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(len(listRes.Infos)).To(Equal(1)) - }) - } - - /* FIXME: related to https://github.com/cs3org/reva/issues/4158 - assertMetadata := func() { - It("sets and unsets metadata", func() { - statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(BeEmpty()) - - By("setting arbitrary metadata") - samRes, err := serviceClient.SetArbitraryMetadata(ctx, &storagep.SetArbitraryMetadataRequest{ - Ref: subdirRef, - ArbitraryMetadata: &storagep.ArbitraryMetadata{Metadata: map[string]string{"foo": "bar"}}, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(samRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(Equal("bar")) - - By("unsetting arbitrary metadata") - uamRes, err := serviceClient.UnsetArbitraryMetadata(ctx, &storagep.UnsetArbitraryMetadataRequest{ - Ref: subdirRef, - ArbitraryMetadataKeys: []string{"foo"}, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(uamRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(BeEmpty()) - }) - } - */ - - Describe("nextcloud", func() { - BeforeEach(func() { - dependencies = map[string]string{ - "storage": "storageprovider-nextcloud.toml", - } - }) - - assertCreateHome() - - Context("with a home and a subdirectory", func() { - JustBeforeEach(func() { - res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - - assertCreateContainer() - assertListContainer() - assertGetPath() - assertDelete() - assertMove() - assertGrants() - assertUploads() - assertDownloads() - assertRecycle() - assertReferences() - // assertMetadata() - }) - - Context("with an existing file /versioned_file", func() { - JustBeforeEach(func() { - fs, err := ocis.New(ctx, map[string]interface{}{ - "root": revads["storage"].TmpRoot, - "enable_home": true, - }) - Expect(err).ToNot(HaveOccurred()) - - content1 := []byte("1") - content2 := []byte("22") - - ctx := ctxpkg.ContextSetUser(context.Background(), user) - - err = fs.CreateHome(ctx) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content1) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content2) - Expect(err).ToNot(HaveOccurred()) - }) - - assertFileVersions() - }) - }) - - Describe("ocis", func() { - BeforeEach(func() { - dependencies = map[string]string{ - "storage": "storageprovider-ocis.toml", - } - }) - - assertCreateHome() - - Context("with a home and a subdirectory", func() { - JustBeforeEach(func() { - res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - - assertCreateContainer() - assertListContainer() - assertGetPath() - assertDelete() - assertMove() - assertGrants() - assertUploads() - assertDownloads() - assertRecycle() - assertReferences() - // assertMetadata() - }) - - Context("with an existing file /versioned_file", func() { - JustBeforeEach(func() { - fs, err := ocis.New(ctx, map[string]interface{}{ - "root": revads["storage"].TmpRoot, - "enable_home": true, - }) - Expect(err).ToNot(HaveOccurred()) - - content1 := []byte("1") - content2 := []byte("22") - - ctx := ctxpkg.ContextSetUser(context.Background(), user) - - err = fs.CreateHome(ctx) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content1) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content2) - Expect(err).ToNot(HaveOccurred()) - }) - - assertFileVersions() - }) - }) - - Describe("owncloud", func() { - BeforeEach(func() { - dependencies = map[string]string{ - "users": "userprovider-json.toml", - "storage": "storageprovider-owncloud.toml", - } - - redisAddress := os.Getenv("REDIS_ADDRESS") - if redisAddress == "" { - Fail("REDIS_ADDRESS not set") - } - variables = map[string]string{ - "redis_address": redisAddress, - } - }) - - assertCreateHome() - - Context("with a home and a subdirectory", func() { - JustBeforeEach(func() { - res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - - subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) - Expect(err).ToNot(HaveOccurred()) - Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) - }) - - assertCreateContainer() - assertListContainer() - assertDelete() - assertMove() - assertGrants() - assertUploads() - assertDownloads() - assertRecycle() - assertReferences() - // assertMetadata() - }) - - Context("with an existing file /versioned_file", func() { - JustBeforeEach(func() { - fs, err := owncloud.New(ctx, map[string]interface{}{ - "datadirectory": revads["storage"].TmpRoot, - "userprovidersvc": revads["users"].GrpcAddress, - "enable_home": true, - }) - Expect(err).ToNot(HaveOccurred()) - - content1 := []byte("1") - content2 := []byte("22") - - ctx := ctxpkg.ContextSetUser(context.Background(), user) - - err = fs.CreateHome(ctx) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content1) - Expect(err).ToNot(HaveOccurred()) - err = helpers.Upload(ctx, fs, versionedFileRef, content2) - Expect(err).ToNot(HaveOccurred()) - }) - - assertFileVersions() - }) - }) -}) From fcf903040378d8dab92ca536fb50b882d4726256 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 13 Sep 2023 15:04:13 +0200 Subject: [PATCH 06/25] wire http traffic --- cmd/revad/runtime/grpc.go | 3 ++ examples/helloworld/revad.toml | 8 ++-- go.mod | 20 +++++--- go.sum | 43 +++++++++++++---- internal/grpc/interceptors/appctx/appctx.go | 5 ++ internal/grpc/interceptors/trace/trace.go | 36 ++++++++++++++ internal/http/interceptors/appctx/appctx.go | 3 ++ pkg/appctx/appctx.go | 14 ++---- pkg/rhttp/rhttp.go | 2 + pkg/trace/trace.go | 53 +++++++++++++++++++++ 10 files changed, 158 insertions(+), 29 deletions(-) create mode 100644 internal/grpc/interceptors/trace/trace.go create mode 100644 pkg/trace/trace.go diff --git a/cmd/revad/runtime/grpc.go b/cmd/revad/runtime/grpc.go index a41309cfa4..65e1dd791d 100644 --- a/cmd/revad/runtime/grpc.go +++ b/cmd/revad/runtime/grpc.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/internal/grpc/interceptors/metrics" "github.com/cs3org/reva/internal/grpc/interceptors/recovery" "github.com/cs3org/reva/internal/grpc/interceptors/token" + "github.com/cs3org/reva/internal/grpc/interceptors/trace" "github.com/cs3org/reva/internal/grpc/interceptors/useragent" "github.com/cs3org/reva/pkg/rgrpc" "github.com/pkg/errors" @@ -81,6 +82,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, } unaryInterceptors = append([]grpc.UnaryServerInterceptor{ + trace.NewUnary(), metrics.NewUnary(), appctx.NewUnary(*logger), token.NewUnary(), @@ -125,6 +127,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, } streamInterceptors = append([]grpc.StreamServerInterceptor{ + trace.NewStream(), metrics.NewStream(), appctx.NewStream(*logger), token.NewStream(), diff --git a/examples/helloworld/revad.toml b/examples/helloworld/revad.toml index f567c8931a..c789d4eaaa 100644 --- a/examples/helloworld/revad.toml +++ b/examples/helloworld/revad.toml @@ -1,10 +1,12 @@ +[http] +address = "localhost:8080" + [http.services.helloworld] -[grpc.services.helloworld] [http.services.prometheus] -[http] -address = "localhost:8080" [grpc] address = "localhost:8081" enable_reflection = true + +[grpc.services.helloworld] diff --git a/go.mod b/go.mod index fa23c57abe..db7dafb587 100644 --- a/go.mod +++ b/go.mod @@ -59,13 +59,18 @@ require ( github.com/wk8/go-ordered-map v1.0.0 go-micro.dev/v4 v4.3.1-0.20211108085239-0c2041e43908 go.opencensus.io v0.24.0 - go.opentelemetry.io/otel/trace v1.11.2 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 + go.opentelemetry.io/otel v1.18.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 + go.opentelemetry.io/otel/sdk v1.18.0 + go.opentelemetry.io/otel/trace v1.18.0 go.step.sm/crypto v0.35.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.13.0 golang.org/x/oauth2 v0.11.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.11.0 - golang.org/x/term v0.11.0 + golang.org/x/sys v0.12.0 + golang.org/x/term v0.12.0 golang.org/x/text v0.13.0 google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 google.golang.org/grpc v1.58.0 @@ -87,10 +92,13 @@ require ( github.com/dolthub/vitess v0.0.0-20221031111135-9aad77e7b39f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.13.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-kit/kit v0.10.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect github.com/gocraft/dbr/v2 v2.7.2 // indirect @@ -142,9 +150,9 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.mongodb.org/mongo-driver v1.11.0 // indirect - go.opentelemetry.io/otel v1.11.2 // indirect + go.opentelemetry.io/otel/metric v1.18.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 65ee0f384e..c4af5e949d 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,7 @@ cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= @@ -212,10 +213,12 @@ cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= @@ -923,6 +926,7 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -1012,6 +1016,7 @@ github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exoscale/egoscale v0.46.0/go.mod h1:mpEXBpROAa/2i5GC0r33rfxG+TxSEka11g1PIXt9+zc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -1020,6 +1025,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -1075,7 +1082,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= @@ -1825,10 +1836,20 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= -go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= -go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= -go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0 h1:b8xjZxHbLrXAum4SxJd1Rlm7Y/fKaB+6ACI7/e5EfSA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0/go.mod h1:1ei0a32xOGkFoySu7y1DAHfcuIhC0pNZpvY2huXuMy4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= +go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 h1:hSWWvDjXHVLq9DkmB+77fl8v7+t+yYiS+eNkiplDK54= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0/go.mod h1:zG7KQql1WjZCaUJd+L/ReSYx4bjbYJxg5ws9ws+mYes= +go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= +go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= +go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= +go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= +go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= +go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1880,8 +1901,9 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2021,8 +2043,9 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2209,8 +2232,9 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2225,8 +2249,9 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/grpc/interceptors/appctx/appctx.go b/internal/grpc/interceptors/appctx/appctx.go index 66827a295c..8bb2f3c780 100644 --- a/internal/grpc/interceptors/appctx/appctx.go +++ b/internal/grpc/interceptors/appctx/appctx.go @@ -22,6 +22,7 @@ import ( "context" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" "google.golang.org/grpc" ) @@ -29,6 +30,8 @@ import ( // NewUnary returns a new unary interceptor that creates the application context. func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + traceID := trace.GetTraceID(ctx) + log = log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) res, err := handler(ctx, req) return res, err @@ -41,6 +44,8 @@ func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { func NewStream(log zerolog.Logger) grpc.StreamServerInterceptor { interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() + traceID := trace.GetTraceID(ctx) + log = log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) wrapped := newWrappedServerStream(ctx, ss) diff --git a/internal/grpc/interceptors/trace/trace.go b/internal/grpc/interceptors/trace/trace.go new file mode 100644 index 0000000000..12cbcc1074 --- /dev/null +++ b/internal/grpc/interceptors/trace/trace.go @@ -0,0 +1,36 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package trace + +import ( + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" +) + +// NewUnary returns a new unary interceptor that adds +// the useragent to the context. +func NewUnary() grpc.UnaryServerInterceptor { + return otelgrpc.UnaryServerInterceptor() +} + +// NewStream returns a new server stream interceptor +// that adds the user agent to the context. +func NewStream() grpc.StreamServerInterceptor { + return otelgrpc.StreamServerInterceptor() +} diff --git a/internal/http/interceptors/appctx/appctx.go b/internal/http/interceptors/appctx/appctx.go index 0ceb43f9bc..6da6dfc2f2 100644 --- a/internal/http/interceptors/appctx/appctx.go +++ b/internal/http/interceptors/appctx/appctx.go @@ -25,6 +25,7 @@ import ( "net/http" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" ) @@ -40,6 +41,8 @@ func New(log zerolog.Logger) func(http.Handler) http.Handler { func handler(log zerolog.Logger, h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + traceID := trace.GetTraceID(ctx) + log = log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) r = r.WithContext(ctx) h.ServeHTTP(w, r) diff --git a/pkg/appctx/appctx.go b/pkg/appctx/appctx.go index 683f48beb7..58b5d56636 100644 --- a/pkg/appctx/appctx.go +++ b/pkg/appctx/appctx.go @@ -20,17 +20,14 @@ package appctx import ( "context" - + "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" - "go.opentelemetry.io/otel/trace" ) -const traceIDKey = "traceid" - // WithLogger returns a context with an associated logger. func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { - traceID := GetTraceID(ctx) - sublog := l.With().Str(traceIDKey, traceID.String()).Logger() + traceID := trace.GetTraceID(ctx) + sublog := l.With().Str(trace.TraceIDKey, traceID).Logger() return sublog.WithContext(ctx) } @@ -39,8 +36,3 @@ func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { func GetLogger(ctx context.Context) *zerolog.Logger { return zerolog.Ctx(ctx) } - -func GetTraceID(ctx context.Context) trace.TraceID { - traceID := trace.SpanContextFromContext(ctx).TraceID() - return traceID -} diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index b516814a6e..58ad42c62f 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -32,6 +32,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/pkg/errors" "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) type Config func(*Server) @@ -277,6 +278,7 @@ func (s *Server) getHandler() (http.Handler, error) { handler = m(handler) } + handler = otelhttp.NewHandler(handler, "http") return handler, nil } diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go new file mode 100644 index 0000000000..f8edc98bd3 --- /dev/null +++ b/pkg/trace/trace.go @@ -0,0 +1,53 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package trace + +import ( + "context" + "go.opentelemetry.io/otel" + stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +const TraceIDKey = "traceid" + +func init() { + // Set up OTLP tracing (stdout for debug). + exporter, err := stdout.New(stdout.WithPrettyPrint()) + if err != nil { + panic(err) + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithBatcher(exporter), + ) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + defer func() { _ = exporter.Shutdown(context.Background()) }() + +} + +func GetTraceID(ctx context.Context) string { + if span := trace.SpanContextFromContext(ctx); span.IsSampled() { + return span.TraceID().String() + } + return "unknown" +} From 16b297eac091d42f1176cc4bfe1a175db450cd39 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 13 Sep 2023 15:21:22 +0200 Subject: [PATCH 07/25] intrument outgoing grpc calls --- pkg/rgrpc/todo/pool/pool.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 67904d0328..644efab196 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -40,6 +40,7 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storageregistry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -90,6 +91,12 @@ func NewConn(options Options) (*grpc.ClientConn, error) { grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(options.MaxCallRecvMsgSize), ), + grpc.WithChainUnaryInterceptor( + otelgrpc.UnaryClientInterceptor(), + ), + grpc.WithChainStreamInterceptor( + otelgrpc.StreamClientInterceptor(), + ), ) if err != nil { return nil, err From 28c69bb465466f8868da99779f1a362f3513af1c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 13 Sep 2023 15:21:22 +0200 Subject: [PATCH 08/25] add stuff --- cmd/revad/runtime/grpc.go | 3 +- cmd/revad/runtime/http.go | 4 +- examples/helloworld/revad.toml | 3 + examples/pingpong/revad.toml | 12 + go.mod | 6 +- go.sum | 12 +- internal/grpc/interceptors/appctx/appctx.go | 8 +- internal/grpc/interceptors/trace/trace.go | 54 ++- internal/grpc/services/loader/loader.go | 1 + internal/grpc/services/pingpong/pingpong.go | 110 ++++++ .../services/pingpong/proto/pingpong.pb.go | 363 ++++++++++++++++++ .../services/pingpong/proto/pingpong.proto | 46 +++ .../pingpong/proto/pingpong_grpc.pb.go | 141 +++++++ internal/http/interceptors/appctx/appctx.go | 4 +- internal/http/interceptors/trace/trace.go | 61 +++ internal/http/services/loader/loader.go | 1 + internal/http/services/pingpong/pingpong.go | 140 +++++++ pkg/appctx/appctx.go | 5 +- pkg/ctx/logger.go | 39 ++ pkg/httpclient/httpclient.go | 60 +++ pkg/rgrpc/status/status.go | 5 +- pkg/rgrpc/todo/pool/pool.go | 8 - pkg/trace/trace.go | 39 +- 23 files changed, 1058 insertions(+), 67 deletions(-) create mode 100644 examples/pingpong/revad.toml create mode 100644 internal/grpc/services/pingpong/pingpong.go create mode 100644 internal/grpc/services/pingpong/proto/pingpong.pb.go create mode 100644 internal/grpc/services/pingpong/proto/pingpong.proto create mode 100644 internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go create mode 100644 internal/http/interceptors/trace/trace.go create mode 100644 internal/http/services/pingpong/pingpong.go create mode 100644 pkg/ctx/logger.go create mode 100644 pkg/httpclient/httpclient.go diff --git a/cmd/revad/runtime/grpc.go b/cmd/revad/runtime/grpc.go index 65e1dd791d..c71b2f5ee5 100644 --- a/cmd/revad/runtime/grpc.go +++ b/cmd/revad/runtime/grpc.go @@ -75,7 +75,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, return nil, nil, errors.Wrap(err, "error creating unary auth interceptor") } - unaryInterceptors := []grpc.UnaryServerInterceptor{authUnary} + unaryInterceptors := []grpc.UnaryServerInterceptor{} for _, t := range unaryTriples { unaryInterceptors = append(unaryInterceptors, t.Interceptor) logger.Info().Msgf("rgrpc: chaining grpc unary interceptor %s with priority %d", t.Name, t.Priority) @@ -89,6 +89,7 @@ func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, useragent.NewUnary(), log.NewUnary(), recovery.NewUnary(), + authUnary, }, unaryInterceptors...) streamTriples := []*streamInterceptorTriple{} diff --git a/cmd/revad/runtime/http.go b/cmd/revad/runtime/http.go index 3af6d00369..5cbb2db9f4 100644 --- a/cmd/revad/runtime/http.go +++ b/cmd/revad/runtime/http.go @@ -26,6 +26,7 @@ import ( "github.com/cs3org/reva/internal/http/interceptors/auth" "github.com/cs3org/reva/internal/http/interceptors/log" "github.com/cs3org/reva/internal/http/interceptors/metrics" + "github.com/cs3org/reva/internal/http/interceptors/trace" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -68,10 +69,11 @@ func initHTTPMiddlewares(conf map[string]map[string]any, unprotected []string, l } middlewares := []global.Middleware{ - metrics.New(), authMiddle, log.New(), appctx.New(*logger), + metrics.New(), + trace.New(), } for _, triple := range triples { diff --git a/examples/helloworld/revad.toml b/examples/helloworld/revad.toml index c789d4eaaa..e034fb8ebb 100644 --- a/examples/helloworld/revad.toml +++ b/examples/helloworld/revad.toml @@ -1,3 +1,6 @@ +[log] +mode = "json" + [http] address = "localhost:8080" diff --git a/examples/pingpong/revad.toml b/examples/pingpong/revad.toml new file mode 100644 index 0000000000..159812a7a1 --- /dev/null +++ b/examples/pingpong/revad.toml @@ -0,0 +1,12 @@ +[log] + +[http] +address = "localhost:8080" + +[http.services.pingpong] + +[grpc] +address = "localhost:8081" +enable_reflection = true + +[grpc.services.pingpong] diff --git a/go.mod b/go.mod index db7dafb587..7b57e9b818 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.11.2 github.com/go-sql-driver/mysql v1.7.1 + github.com/gofrs/uuid v4.3.1+incompatible github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.3 github.com/gomodule/redigo v1.8.9 @@ -59,11 +60,7 @@ require ( github.com/wk8/go-ordered-map v1.0.0 go-micro.dev/v4 v4.3.1-0.20211108085239-0c2041e43908 go.opencensus.io v0.24.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 - go.opentelemetry.io/otel v1.18.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 - go.opentelemetry.io/otel/sdk v1.18.0 go.opentelemetry.io/otel/trace v1.18.0 go.step.sm/crypto v0.35.0 golang.org/x/crypto v0.13.0 @@ -150,6 +147,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.mongodb.org/mongo-driver v1.11.0 // indirect + go.opentelemetry.io/otel v1.18.0 // indirect go.opentelemetry.io/otel/metric v1.18.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.15.0 // indirect diff --git a/go.sum b/go.sum index c4af5e949d..5125059f7a 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= @@ -213,12 +212,10 @@ cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/ cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= @@ -926,7 +923,6 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -1016,7 +1012,6 @@ github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exoscale/egoscale v0.46.0/go.mod h1:mpEXBpROAa/2i5GC0r33rfxG+TxSEka11g1PIXt9+zc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -1121,6 +1116,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -1836,18 +1832,12 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0 h1:b8xjZxHbLrXAum4SxJd1Rlm7Y/fKaB+6ACI7/e5EfSA= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0/go.mod h1:1ei0a32xOGkFoySu7y1DAHfcuIhC0pNZpvY2huXuMy4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 h1:hSWWvDjXHVLq9DkmB+77fl8v7+t+yYiS+eNkiplDK54= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0/go.mod h1:zG7KQql1WjZCaUJd+L/ReSYx4bjbYJxg5ws9ws+mYes= go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= -go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= -go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= diff --git a/internal/grpc/interceptors/appctx/appctx.go b/internal/grpc/interceptors/appctx/appctx.go index 8bb2f3c780..39a8a410d8 100644 --- a/internal/grpc/interceptors/appctx/appctx.go +++ b/internal/grpc/interceptors/appctx/appctx.go @@ -30,8 +30,8 @@ import ( // NewUnary returns a new unary interceptor that creates the application context. func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - traceID := trace.GetTraceID(ctx) - log = log.With().Str("traceid", traceID).Logger() + traceID := trace.Get(ctx) + log = log.With().Str(trace.Key, traceID).Logger() ctx = appctx.WithLogger(ctx, &log) res, err := handler(ctx, req) return res, err @@ -44,8 +44,8 @@ func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { func NewStream(log zerolog.Logger) grpc.StreamServerInterceptor { interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() - traceID := trace.GetTraceID(ctx) - log = log.With().Str("traceid", traceID).Logger() + traceID := trace.Get(ctx) + log = log.With().Str(trace.Key, traceID).Logger() ctx = appctx.WithLogger(ctx, &log) wrapped := newWrappedServerStream(ctx, ss) diff --git a/internal/grpc/interceptors/trace/trace.go b/internal/grpc/interceptors/trace/trace.go index 12cbcc1074..2f94c6a3bb 100644 --- a/internal/grpc/interceptors/trace/trace.go +++ b/internal/grpc/interceptors/trace/trace.go @@ -19,18 +19,62 @@ package trace import ( - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "context" + + "github.com/cs3org/reva/pkg/trace" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) +func getContext(ctx context.Context) context.Context { + var traceID string + md, ok := metadata.FromIncomingContext(ctx) + if ok && md != nil { + if val, ok := md[trace.Key]; ok { + if len(val) > 0 && val[0] != "" { + traceID = val[0] + } + } + } + + if traceID == "" { + traceID = trace.Generate() + } + + ctx = trace.Set(ctx, traceID) + return ctx +} + // NewUnary returns a new unary interceptor that adds -// the useragent to the context. +// trace information for the request. func NewUnary() grpc.UnaryServerInterceptor { - return otelgrpc.UnaryServerInterceptor() + interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ctx = getContext(ctx) + return handler(ctx, req) + } + return interceptor } // NewStream returns a new server stream interceptor -// that adds the user agent to the context. +// that adds trace information to the request. func NewStream() grpc.StreamServerInterceptor { - return otelgrpc.StreamServerInterceptor() + interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + ctx := getContext(ss.Context()) + wrapped := newWrappedServerStream(ctx, ss) + return handler(srv, wrapped) + } + return interceptor +} + +func newWrappedServerStream(ctx context.Context, ss grpc.ServerStream) *wrappedServerStream { + return &wrappedServerStream{ServerStream: ss, newCtx: ctx} +} + +type wrappedServerStream struct { + grpc.ServerStream + newCtx context.Context +} + +func (ss *wrappedServerStream) Context() context.Context { + return ss.newCtx } diff --git a/internal/grpc/services/loader/loader.go b/internal/grpc/services/loader/loader.go index 801acd285f..c028346e0f 100644 --- a/internal/grpc/services/loader/loader.go +++ b/internal/grpc/services/loader/loader.go @@ -34,6 +34,7 @@ import ( _ "github.com/cs3org/reva/internal/grpc/services/ocmproviderauthorizer" _ "github.com/cs3org/reva/internal/grpc/services/ocmshareprovider" _ "github.com/cs3org/reva/internal/grpc/services/permissions" + _ "github.com/cs3org/reva/internal/grpc/services/pingpong" _ "github.com/cs3org/reva/internal/grpc/services/preferences" _ "github.com/cs3org/reva/internal/grpc/services/publicshareprovider" _ "github.com/cs3org/reva/internal/grpc/services/publicstorageprovider" diff --git a/internal/grpc/services/pingpong/pingpong.go b/internal/grpc/services/pingpong/pingpong.go new file mode 100644 index 0000000000..284332de15 --- /dev/null +++ b/internal/grpc/services/pingpong/pingpong.go @@ -0,0 +1,110 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities + +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package pingpong + +import ( + "context" + "crypto/tls" + "io/ioutil" + "net/http" + + "github.com/cs3org/reva/internal/grpc/services/pingpong/proto" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/httpclient" + "github.com/cs3org/reva/pkg/rgrpc" + "github.com/cs3org/reva/pkg/utils/cfg" + "google.golang.org/grpc" +) + +func init() { + rgrpc.Register("pingpong", New) +} + +type conf struct { + Endpoint string +} + +type service struct { + conf *conf + *proto.UnimplementedPingPongServiceServer +} + +func (c *conf) ApplyDefaults() { + if c.Endpoint == "" { + c.Endpoint = "http://localhost:8080/pingpong" + } +} + +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { + var c conf + if err := cfg.Decode(m, &c); err != nil { + return nil, err + } + service := &service{conf: &c} + return service, nil +} + +func (s *service) Close() error { + return nil +} + +func (s *service) UnprotectedEndpoints() []string { + return []string{"/revad.pingpong.PingPongService/Ping", "/revad.pingpong.PingPongService/Pong"} +} + +func (s *service) Register(ss *grpc.Server) { + proto.RegisterPingPongServiceServer(ss, s) +} + +func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingResponse, error) { + log := appctx.GetLogger(ctx) + + // we call the http Pong method + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + client := httpclient.NewWithRoundTripper(tr) + req, err := httpclient.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) + if err != nil { + log.Error().Err(err).Msg("error creating http request") + return nil, err + } + + res, err := client.Do(req) + if err != nil { + log.Error().Err(err).Msg("eror doing http pong") + return nil, err + } + + defer res.Body.Close() + data, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Error().Err(err).Msg("error reading response body") + return nil, err + } + + return &proto.PingResponse{Info: string(data)}, nil +} + +func (s *service) Pong(_ context.Context, _ *proto.PongRequest) (*proto.PongResponse, error) { + res := &proto.PongResponse{Info: "pong"} + return res, nil +} diff --git a/internal/grpc/services/pingpong/proto/pingpong.pb.go b/internal/grpc/services/pingpong/proto/pingpong.pb.go new file mode 100644 index 0000000000..aac5826968 --- /dev/null +++ b/internal/grpc/services/pingpong/proto/pingpong.pb.go @@ -0,0 +1,363 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.7.1 +// source: pingpong.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type PingRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info string `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PingRequest) Reset() { + *x = PingRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pingpong_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PingRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PingRequest) ProtoMessage() {} + +func (x *PingRequest) ProtoReflect() protoreflect.Message { + mi := &file_pingpong_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. +func (*PingRequest) Descriptor() ([]byte, []int) { + return file_pingpong_proto_rawDescGZIP(), []int{0} +} + +func (x *PingRequest) GetInfo() string { + if x != nil { + return x.Info + } + return "" +} + +type PingResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info string `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PingResponse) Reset() { + *x = PingResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pingpong_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PingResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PingResponse) ProtoMessage() {} + +func (x *PingResponse) ProtoReflect() protoreflect.Message { + mi := &file_pingpong_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. +func (*PingResponse) Descriptor() ([]byte, []int) { + return file_pingpong_proto_rawDescGZIP(), []int{1} +} + +func (x *PingResponse) GetInfo() string { + if x != nil { + return x.Info + } + return "" +} + +type PongRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info string `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PongRequest) Reset() { + *x = PongRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pingpong_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PongRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PongRequest) ProtoMessage() {} + +func (x *PongRequest) ProtoReflect() protoreflect.Message { + mi := &file_pingpong_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PongRequest.ProtoReflect.Descriptor instead. +func (*PongRequest) Descriptor() ([]byte, []int) { + return file_pingpong_proto_rawDescGZIP(), []int{2} +} + +func (x *PongRequest) GetInfo() string { + if x != nil { + return x.Info + } + return "" +} + +type PongResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Info string `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"` +} + +func (x *PongResponse) Reset() { + *x = PongResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pingpong_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PongResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PongResponse) ProtoMessage() {} + +func (x *PongResponse) ProtoReflect() protoreflect.Message { + mi := &file_pingpong_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PongResponse.ProtoReflect.Descriptor instead. +func (*PongResponse) Descriptor() ([]byte, []int) { + return file_pingpong_proto_rawDescGZIP(), []int{3} +} + +func (x *PongResponse) GetInfo() string { + if x != nil { + return x.Info + } + return "" +} + +var File_pingpong_proto protoreflect.FileDescriptor + +var file_pingpong_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0e, 0x72, 0x65, 0x76, 0x61, 0x64, 0x2e, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, + 0x22, 0x21, 0x0a, 0x0b, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, + 0x6e, 0x66, 0x6f, 0x22, 0x22, 0x0a, 0x0c, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x21, 0x0a, 0x0b, 0x50, 0x6f, 0x6e, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x22, 0x0a, 0x0c, 0x50, 0x6f, + 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x32, 0x97, + 0x01, 0x0a, 0x0f, 0x50, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1b, 0x2e, 0x72, 0x65, 0x76, + 0x61, 0x64, 0x2e, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, 0x2e, 0x50, 0x69, 0x6e, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x65, 0x76, 0x61, 0x64, 0x2e, + 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x50, 0x6f, 0x6e, 0x67, 0x12, 0x1b, 0x2e, + 0x72, 0x65, 0x76, 0x61, 0x64, 0x2e, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, 0x2e, 0x50, + 0x6f, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x65, 0x76, + 0x61, 0x64, 0x2e, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, 0x6e, 0x67, 0x2e, 0x50, 0x6f, 0x6e, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x73, 0x33, 0x6f, 0x72, 0x67, 0x2f, 0x72, 0x65, + 0x76, 0x61, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x70, 0x69, 0x6e, 0x67, 0x70, 0x6f, + 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_pingpong_proto_rawDescOnce sync.Once + file_pingpong_proto_rawDescData = file_pingpong_proto_rawDesc +) + +func file_pingpong_proto_rawDescGZIP() []byte { + file_pingpong_proto_rawDescOnce.Do(func() { + file_pingpong_proto_rawDescData = protoimpl.X.CompressGZIP(file_pingpong_proto_rawDescData) + }) + return file_pingpong_proto_rawDescData +} + +var file_pingpong_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pingpong_proto_goTypes = []interface{}{ + (*PingRequest)(nil), // 0: revad.pingpong.PingRequest + (*PingResponse)(nil), // 1: revad.pingpong.PingResponse + (*PongRequest)(nil), // 2: revad.pingpong.PongRequest + (*PongResponse)(nil), // 3: revad.pingpong.PongResponse +} +var file_pingpong_proto_depIdxs = []int32{ + 0, // 0: revad.pingpong.PingPongService.Ping:input_type -> revad.pingpong.PingRequest + 2, // 1: revad.pingpong.PingPongService.Pong:input_type -> revad.pingpong.PongRequest + 1, // 2: revad.pingpong.PingPongService.Ping:output_type -> revad.pingpong.PingResponse + 3, // 3: revad.pingpong.PingPongService.Pong:output_type -> revad.pingpong.PongResponse + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_pingpong_proto_init() } +func file_pingpong_proto_init() { + if File_pingpong_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_pingpong_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PingRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pingpong_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PingResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pingpong_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PongRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pingpong_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PongResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_pingpong_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_pingpong_proto_goTypes, + DependencyIndexes: file_pingpong_proto_depIdxs, + MessageInfos: file_pingpong_proto_msgTypes, + }.Build() + File_pingpong_proto = out.File + file_pingpong_proto_rawDesc = nil + file_pingpong_proto_goTypes = nil + file_pingpong_proto_depIdxs = nil +} diff --git a/internal/grpc/services/pingpong/proto/pingpong.proto b/internal/grpc/services/pingpong/proto/pingpong.proto new file mode 100644 index 0000000000..176800aa94 --- /dev/null +++ b/internal/grpc/services/pingpong/proto/pingpong.proto @@ -0,0 +1,46 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +syntax = "proto3"; + +package revad.pingpong; + +option go_package = "github.com/cs3org/reva/grpc/internal/services/pingpong/proto"; + +service PingPongService { + rpc Ping(PingRequest) returns (PingResponse); + rpc Pong(PongRequest) returns (PongResponse); +} + +message PingRequest { + string info = 1; +} + +message PingResponse { + string info = 1; +} +message PongRequest { + string info = 1; +} + +message PongResponse { + string info = 1; +} + +// to compile this into grpc, cd in the directory where this file lives and execute: +// protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pingpong.proto diff --git a/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go b/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go new file mode 100644 index 0000000000..26f6e066a6 --- /dev/null +++ b/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go @@ -0,0 +1,141 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.7.1 +// source: pingpong.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// PingPongServiceClient is the client API for PingPongService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type PingPongServiceClient interface { + Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) + Pong(ctx context.Context, in *PongRequest, opts ...grpc.CallOption) (*PongResponse, error) +} + +type pingPongServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewPingPongServiceClient(cc grpc.ClientConnInterface) PingPongServiceClient { + return &pingPongServiceClient{cc} +} + +func (c *pingPongServiceClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { + out := new(PingResponse) + err := c.cc.Invoke(ctx, "/revad.pingpong.PingPongService/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pingPongServiceClient) Pong(ctx context.Context, in *PongRequest, opts ...grpc.CallOption) (*PongResponse, error) { + out := new(PongResponse) + err := c.cc.Invoke(ctx, "/revad.pingpong.PingPongService/Pong", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// PingPongServiceServer is the server API for PingPongService service. +// All implementations must embed UnimplementedPingPongServiceServer +// for forward compatibility +type PingPongServiceServer interface { + Ping(context.Context, *PingRequest) (*PingResponse, error) + Pong(context.Context, *PongRequest) (*PongResponse, error) + mustEmbedUnimplementedPingPongServiceServer() +} + +// UnimplementedPingPongServiceServer must be embedded to have forward compatible implementations. +type UnimplementedPingPongServiceServer struct { +} + +func (UnimplementedPingPongServiceServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedPingPongServiceServer) Pong(context.Context, *PongRequest) (*PongResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Pong not implemented") +} +func (UnimplementedPingPongServiceServer) mustEmbedUnimplementedPingPongServiceServer() {} + +// UnsafePingPongServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to PingPongServiceServer will +// result in compilation errors. +type UnsafePingPongServiceServer interface { + mustEmbedUnimplementedPingPongServiceServer() +} + +func RegisterPingPongServiceServer(s grpc.ServiceRegistrar, srv PingPongServiceServer) { + s.RegisterService(&PingPongService_ServiceDesc, srv) +} + +func _PingPongService_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PingPongServiceServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/revad.pingpong.PingPongService/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PingPongServiceServer).Ping(ctx, req.(*PingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PingPongService_Pong_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PongRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PingPongServiceServer).Pong(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/revad.pingpong.PingPongService/Pong", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PingPongServiceServer).Pong(ctx, req.(*PongRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// PingPongService_ServiceDesc is the grpc.ServiceDesc for PingPongService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var PingPongService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "revad.pingpong.PingPongService", + HandlerType: (*PingPongServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Ping", + Handler: _PingPongService_Ping_Handler, + }, + { + MethodName: "Pong", + Handler: _PingPongService_Pong_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "pingpong.proto", +} diff --git a/internal/http/interceptors/appctx/appctx.go b/internal/http/interceptors/appctx/appctx.go index 6da6dfc2f2..faf994135b 100644 --- a/internal/http/interceptors/appctx/appctx.go +++ b/internal/http/interceptors/appctx/appctx.go @@ -41,8 +41,8 @@ func New(log zerolog.Logger) func(http.Handler) http.Handler { func handler(log zerolog.Logger, h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - traceID := trace.GetTraceID(ctx) - log = log.With().Str("traceid", traceID).Logger() + traceID := trace.Get(ctx) + log = log.With().Str(trace.Key, traceID).Logger() ctx = appctx.WithLogger(ctx, &log) r = r.WithContext(ctx) h.ServeHTTP(w, r) diff --git a/internal/http/interceptors/trace/trace.go b/internal/http/interceptors/trace/trace.go new file mode 100644 index 0000000000..25a83a4e9a --- /dev/null +++ b/internal/http/interceptors/trace/trace.go @@ -0,0 +1,61 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package appctx creates a context with useful +// components attached to the context like loggers and +// token managers. +package trace + +import ( + "net/http" + + "github.com/cs3org/reva/pkg/trace" + "google.golang.org/grpc/metadata" +) + +// New returns a new HTTP middleware that stores the log +// in the context with request ID information. +func New() func(http.Handler) http.Handler { + chain := func(h http.Handler) http.Handler { + return handler(h) + } + return chain +} + +func handler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + // try to get trace from context + traceID := trace.Get(ctx) + if traceID == "" { + // check if traceID is coming from header + traceID = r.Header.Get("X-Trace-ID") + if traceID == "" { + traceID = trace.Generate() + } + ctx = trace.Set(ctx, traceID) + } + + // in case the http service will call a grpc service, + // we set the outgoing context so the trace information is + // passed through the two protocols. + ctx = metadata.AppendToOutgoingContext(ctx, trace.Key, traceID) + r = r.WithContext(ctx) + h.ServeHTTP(w, r) + }) +} diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index c15d96b68a..e680ad4e3a 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -32,6 +32,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/overleaf" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocs" + _ "github.com/cs3org/reva/internal/http/services/pingpong" _ "github.com/cs3org/reva/internal/http/services/plugins" _ "github.com/cs3org/reva/internal/http/services/preferences" _ "github.com/cs3org/reva/internal/http/services/prometheus" diff --git a/internal/http/services/pingpong/pingpong.go b/internal/http/services/pingpong/pingpong.go new file mode 100644 index 0000000000..7dbca3fd45 --- /dev/null +++ b/internal/http/services/pingpong/pingpong.go @@ -0,0 +1,140 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package pingpong + +import ( + "context" + "net/http" + "strings" + + "github.com/cs3org/reva/internal/grpc/services/pingpong/proto" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/utils/cfg" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func init() { + global.Register("pingpong", New) +} + +// New returns a new helloworld service. +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err + } + + return &svc{conf: &c}, nil +} + +// Close performs cleanup. +func (s *svc) Close() error { + return nil +} + +type config struct { + Prefix string `mapstructure:"prefix"` + Endpoint string `mapstructure:"endpoint"` +} + +func (c *config) ApplyDefaults() { + if c.Prefix == "" { + c.Prefix = "pingpong" + } + + if c.Endpoint == "" { + c.Endpoint = "localhost:8081" + } +} + +type svc struct { + conf *config +} + +func (s *svc) Prefix() string { + return s.conf.Prefix +} + +func (s *svc) Unprotected() []string { + return []string{"/"} +} + +func (s *svc) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + action := strings.TrimPrefix(r.URL.Path, s.conf.Prefix) + log := appctx.GetLogger(r.Context()) + log.Info().Msgf("action: %s", action) + switch action { + case "/ping": + s.doPing(w, r) + return + case "/pong": + s.doPong(w, r) + return + default: + w.WriteHeader(http.StatusBadRequest) + return + } + }) +} + +func (s *svc) getClient() (proto.PingPongServiceClient, error) { + conn, err := grpc.Dial( + s.conf.Endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + + if err != nil { + return nil, err + } + + client := proto.NewPingPongServiceClient(conn) + return client, nil +} + +// doPing will call the grpc Pong method +func (s *svc) doPing(w http.ResponseWriter, r *http.Request) { + log := appctx.GetLogger(r.Context()) + client, err := s.getClient() + if err != nil { + log.Error().Err(err).Msg("error getting grpc client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + pingRes, err := client.Pong(r.Context(), &proto.PongRequest{}) + if err != nil { + log.Error().Err(err).Msg("error doing grpc pong") + w.WriteHeader(http.StatusInternalServerError) + return + } + + log.Info().Msg("pinging from http to grpc") + + w.Write([]byte(pingRes.Info)) + return +} + +// doPong will be (http) called from grpc Pong +func (s *svc) doPong(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("pong")) + return +} diff --git a/pkg/appctx/appctx.go b/pkg/appctx/appctx.go index 58b5d56636..a708956cd3 100644 --- a/pkg/appctx/appctx.go +++ b/pkg/appctx/appctx.go @@ -20,14 +20,15 @@ package appctx import ( "context" + "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" ) // WithLogger returns a context with an associated logger. func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { - traceID := trace.GetTraceID(ctx) - sublog := l.With().Str(trace.TraceIDKey, traceID).Logger() + traceID := trace.Get(ctx) + sublog := l.With().Str(trace.Key, traceID).Logger() return sublog.WithContext(ctx) } diff --git a/pkg/ctx/logger.go b/pkg/ctx/logger.go new file mode 100644 index 0000000000..a6907782b6 --- /dev/null +++ b/pkg/ctx/logger.go @@ -0,0 +1,39 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ctx + +import ( + "context" + + "github.com/cs3org/reva/pkg/trace" + "github.com/rs/zerolog" +) + +// WithLogger returns a context with an associated logger. +func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { + traceID := trace.Get(ctx) + sublog := l.With().Str(trace.Key, traceID).Logger() + return sublog.WithContext(ctx) +} + +// GetLogger returns the logger associated with the given context +// or a disabled logger in case no logger is stored inside the context. +func GetLogger(ctx context.Context) *zerolog.Logger { + return zerolog.Ctx(ctx) +} diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go new file mode 100644 index 0000000000..075c3a4292 --- /dev/null +++ b/pkg/httpclient/httpclient.go @@ -0,0 +1,60 @@ +package httpclient + +import ( + //"io" + "context" + "io" + "net/http" + + "github.com/cs3org/reva/pkg/trace" + //"net/url" +) + +// New creates an http.Client with a custom round tripper that adds tracing headers +// This client must be used in the reva codebase and usage of standard http.Client +func New() *http.Client { + t := &injectTransport{rt: http.DefaultTransport} + client := http.Client{ + Transport: t, + } + return &client +} + +// NewWithRoundTripper works as New but wraps the rt argument with the custom round tripper +func NewWithRoundTripper(rt http.RoundTripper) *http.Client { + t := &injectTransport{rt: rt} + client := http.Client{ + Transport: t, + } + return &client +} + +type injectTransport struct { + rt http.RoundTripper +} + +func (t injectTransport) RoundTrip(r *http.Request) (*http.Response, error) { + // assume the request context has not been populated with tracing information. + ctx := r.Context() + + traceID := trace.Get(ctx) + + r.Header.Add("X-Trace-ID", traceID) + + tkn, ok := appctx.ContextGetToken(ctx) + if ok { + httpReq.Header.Set(appctx.TokenHeader, tkn) + } + + return t.rt.RoundTrip(r) +} + +// NewRequest creates an HTTP request that sets tracing headers +func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) { + httpReq, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + return httpReq, nil +} diff --git a/pkg/rgrpc/status/status.go b/pkg/rgrpc/status/status.go index 27c285bcbc..64b7b4b718 100644 --- a/pkg/rgrpc/status/status.go +++ b/pkg/rgrpc/status/status.go @@ -28,7 +28,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "go.opentelemetry.io/otel/trace" + "github.com/cs3org/reva/pkg/trace" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -216,6 +216,5 @@ func NewErrorFromCode(code rpc.Code, pkgname string) error { // internal function to attach the trace to a context. func getTrace(ctx context.Context) string { - span := trace.SpanFromContext(ctx) - return span.SpanContext().TraceID().String() + return trace.Get(ctx) } diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 644efab196..e8f96129b4 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -40,7 +40,6 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storageregistry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -82,7 +81,6 @@ var ( ) // NewConn creates a new connection to a grpc server -// with open census tracing support. // TODO(labkode): make grpc tls configurable. func NewConn(options Options) (*grpc.ClientConn, error) { conn, err := grpc.Dial( @@ -91,12 +89,6 @@ func NewConn(options Options) (*grpc.ClientConn, error) { grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(options.MaxCallRecvMsgSize), ), - grpc.WithChainUnaryInterceptor( - otelgrpc.UnaryClientInterceptor(), - ), - grpc.WithChainStreamInterceptor( - otelgrpc.StreamClientInterceptor(), - ), ) if err != nil { return nil, err diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index f8edc98bd3..64c9a4ef09 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,39 +15,26 @@ // In applying this license, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. - package trace import ( "context" - "go.opentelemetry.io/otel" - stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/trace" + + "github.com/gofrs/uuid" ) -const TraceIDKey = "traceid" +const Key string = "traceid" -func init() { - // Set up OTLP tracing (stdout for debug). - exporter, err := stdout.New(stdout.WithPrettyPrint()) - if err != nil { - panic(err) - } - tp := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithBatcher(exporter), - ) - otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) - defer func() { _ = exporter.Shutdown(context.Background()) }() +func Get(ctx context.Context) (t string) { + t, _ = ctx.Value(Key).(string) + return +} +func Generate() string { + return uuid.Must(uuid.NewV4()).String() } -func GetTraceID(ctx context.Context) string { - if span := trace.SpanContextFromContext(ctx); span.IsSampled() { - return span.TraceID().String() - } - return "unknown" +// ContextSetTrace stores the trace in the context. +func Set(ctx context.Context, trace string) context.Context { + return context.WithValue(ctx, Key, trace) } From 5d32635725029270b64b8df17e81d141f7131f01 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 26 Sep 2023 16:30:47 +0200 Subject: [PATCH 09/25] merge appctx and ctx --- cmd/reva/download.go | 2 +- cmd/reva/grpc.go | 2 +- cmd/reva/upload.go | 2 +- internal/grpc/interceptors/auth/auth.go | 2 +- internal/grpc/interceptors/auth/scope.go | 2 +- internal/grpc/interceptors/log/log.go | 2 +- internal/grpc/interceptors/token/token.go | 2 +- .../grpc/interceptors/useragent/useragent.go | 2 +- internal/grpc/services/datatx/datatx.go | 2 +- internal/grpc/services/gateway/appprovider.go | 2 +- .../grpc/services/gateway/authprovider.go | 2 +- .../grpc/services/gateway/ocmshareprovider.go | 2 +- .../grpc/services/gateway/storageprovider.go | 2 +- .../services/gateway/usershareprovider.go | 2 +- .../services/gateway/webdavstorageprovider.go | 2 +- .../ocminvitemanager/ocminvitemanager.go | 2 +- .../ocmshareprovider/ocmshareprovider.go | 2 +- .../publicshareprovider.go | 2 +- .../usershareprovider/usershareprovider.go | 2 +- internal/http/interceptors/auth/auth.go | 2 +- .../auth/token/strategy/header/header.go | 2 +- .../tokenwriter/strategy/header/header.go | 2 +- internal/http/services/overleaf/overleaf.go | 2 +- internal/http/services/owncloud/ocdav/dav.go | 2 +- .../http/services/owncloud/ocdav/ocdav.go | 2 +- .../http/services/owncloud/ocdav/propfind.go | 2 +- .../http/services/owncloud/ocdav/proppatch.go | 2 +- .../http/services/owncloud/ocdav/report.go | 2 +- .../http/services/owncloud/ocdav/trashbin.go | 2 +- internal/http/services/owncloud/ocs/cache.go | 2 +- .../ocs/handlers/apps/sharing/shares/group.go | 2 +- .../handlers/apps/sharing/shares/public.go | 2 +- .../handlers/apps/sharing/shares/shares.go | 2 +- .../ocs/handlers/apps/sharing/shares/user.go | 2 +- .../handlers/cloud/capabilities/uploads.go | 2 +- .../owncloud/ocs/handlers/cloud/user/user.go | 2 +- .../ocs/handlers/cloud/users/users.go | 2 +- .../services/reverseproxy/reverseproxy.go | 2 +- internal/http/services/sciencemesh/token.go | 2 +- pkg/app/provider/wopi/wopi.go | 2 +- pkg/appauth/manager/json/json.go | 2 +- pkg/appauth/manager/json/json_test.go | 2 +- pkg/{ctx => appctx}/agentctx.go | 2 +- pkg/{ctx => appctx}/agentctx_test.go | 2 +- pkg/appctx/{appctx.go => logger.go} | 0 pkg/{ctx => appctx}/tokenctx.go | 2 +- pkg/{ctx => appctx}/userctx.go | 2 +- pkg/auth/manager/nextcloud/nextcloud_test.go | 2 +- pkg/cbox/favorite/sql/sql.go | 2 +- pkg/cbox/preferences/sql/sql.go | 2 +- pkg/cbox/share/sql/sql.go | 2 +- .../storage/eoshomewrapper/eoshomewrapper.go | 2 +- pkg/cbox/storage/eoswrapper/eoswrapper.go | 2 +- pkg/ctx/logger.go | 39 ------------------- pkg/eosclient/eosbinary/eosbinary.go | 2 +- pkg/httpclient/httpclient.go | 3 +- .../repository/nextcloud/nextcloud_test.go | 2 +- pkg/ocm/storage/outcoming/ocm.go | 2 +- pkg/preferences/memory/memory.go | 2 +- pkg/rhttp/client.go | 2 +- pkg/sdk/common/net/net.go | 2 +- pkg/share/cache/warmup/cbox/cbox.go | 2 +- pkg/share/manager/json/json.go | 2 +- pkg/share/manager/memory/memory.go | 2 +- pkg/share/manager/sql/sql.go | 2 +- pkg/share/manager/sql/sql_test.go | 2 +- pkg/storage/favorite/memory/memory_test.go | 2 +- pkg/storage/fs/cback/cback.go | 2 +- pkg/storage/fs/cephfs/upload.go | 2 +- pkg/storage/fs/cephfs/user.go | 2 +- pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 +- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/fs/owncloud/upload.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql.go | 2 +- pkg/storage/fs/owncloudsql/upload.go | 2 +- pkg/storage/registry/static/static.go | 2 +- pkg/storage/registry/static/static_test.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/eosfs/eosfs_test.go | 2 +- pkg/storage/utils/localfs/localfs.go | 2 +- pkg/storage/utils/localfs/upload.go | 2 +- pkg/user/manager/nextcloud/nextcloud.go | 2 +- pkg/user/manager/nextcloud/nextcloud_test.go | 2 +- tests/integration/grpc/ocm_invitation_test.go | 2 +- tests/integration/grpc/userprovider_test.go | 2 +- 86 files changed, 85 insertions(+), 123 deletions(-) rename pkg/{ctx => appctx}/agentctx.go (99%) rename pkg/{ctx => appctx}/agentctx_test.go (99%) rename pkg/appctx/{appctx.go => logger.go} (100%) rename pkg/{ctx => appctx}/tokenctx.go (99%) rename pkg/{ctx => appctx}/userctx.go (99%) delete mode 100644 pkg/ctx/logger.go diff --git a/cmd/reva/download.go b/cmd/reva/download.go index b870e57d61..5fb2747e63 100644 --- a/cmd/reva/download.go +++ b/cmd/reva/download.go @@ -29,7 +29,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" diff --git a/cmd/reva/grpc.go b/cmd/reva/grpc.go index 10d0400227..b1a7ad2a86 100644 --- a/cmd/reva/grpc.go +++ b/cmd/reva/grpc.go @@ -26,7 +26,7 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ins "google.golang.org/grpc/credentials/insecure" diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index 09eebc457f..a3056ae51f 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -36,7 +36,7 @@ import ( "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/crypto" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" diff --git a/internal/grpc/interceptors/auth/auth.go b/internal/grpc/interceptors/auth/auth.go index 601a0e2ee3..ee6b384dd3 100644 --- a/internal/grpc/interceptors/auth/auth.go +++ b/internal/grpc/interceptors/auth/auth.go @@ -28,7 +28,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index 50fc65dad2..b1c82122ac 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -37,7 +37,7 @@ import ( registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" statuspkg "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/interceptors/log/log.go b/internal/grpc/interceptors/log/log.go index 3dea1958e7..da114581e2 100644 --- a/internal/grpc/interceptors/log/log.go +++ b/internal/grpc/interceptors/log/log.go @@ -23,7 +23,7 @@ import ( "time" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/rs/zerolog" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/internal/grpc/interceptors/token/token.go b/internal/grpc/interceptors/token/token.go index fe3c62c4c6..2b66a37207 100644 --- a/internal/grpc/interceptors/token/token.go +++ b/internal/grpc/interceptors/token/token.go @@ -21,7 +21,7 @@ package token import ( "context" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) diff --git a/internal/grpc/interceptors/useragent/useragent.go b/internal/grpc/interceptors/useragent/useragent.go index 5f83229308..3c4636fc20 100644 --- a/internal/grpc/interceptors/useragent/useragent.go +++ b/internal/grpc/interceptors/useragent/useragent.go @@ -21,7 +21,7 @@ package useragent import ( "context" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index 662dffded7..497820bc1f 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -24,7 +24,7 @@ import ( ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" txdriver "github.com/cs3org/reva/pkg/datatx" txregistry "github.com/cs3org/reva/pkg/datatx/manager/registry" repoRegistry "github.com/cs3org/reva/pkg/datatx/repository/registry" diff --git a/internal/grpc/services/gateway/appprovider.go b/internal/grpc/services/gateway/appprovider.go index a6b06bcbde..28f17abd00 100644 --- a/internal/grpc/services/gateway/appprovider.go +++ b/internal/grpc/services/gateway/appprovider.go @@ -33,7 +33,7 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/services/gateway/authprovider.go b/internal/grpc/services/gateway/authprovider.go index b462bef6ed..d0a405de62 100644 --- a/internal/grpc/services/gateway/authprovider.go +++ b/internal/grpc/services/gateway/authprovider.go @@ -29,7 +29,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index 5effeab924..8cce496f46 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -30,7 +30,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index fb73680beb..10e70b1c04 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -34,7 +34,7 @@ import ( registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index da83877149..af6dec5ce8 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -29,7 +29,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go index a30da26d79..33c6bb6601 100644 --- a/internal/grpc/services/gateway/webdavstorageprovider.go +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -28,7 +28,7 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/pkg/errors" "github.com/studio-b12/gowebdav" diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index 0012d0e3dd..c4ae02047f 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -27,7 +27,7 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/invite" diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 7f45e1dfdf..abf3ff7b17 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -35,7 +35,7 @@ import ( providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/ocmd" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/share" diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index 09313d2f14..ea7454b68c 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -25,7 +25,7 @@ import ( link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/publicshare" diff --git a/internal/grpc/services/usershareprovider/usershareprovider.go b/internal/grpc/services/usershareprovider/usershareprovider.go index 3de8bfa3c8..f917af9445 100644 --- a/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/internal/grpc/services/usershareprovider/usershareprovider.go @@ -26,7 +26,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index 56d9023965..ee8470ce0f 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -35,7 +35,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/http/interceptors/auth/token/strategy/header/header.go b/internal/http/interceptors/auth/token/strategy/header/header.go index c0acb13e36..558ea3eba3 100644 --- a/internal/http/interceptors/auth/token/strategy/header/header.go +++ b/internal/http/interceptors/auth/token/strategy/header/header.go @@ -23,7 +23,7 @@ import ( "github.com/cs3org/reva/internal/http/interceptors/auth/token/registry" "github.com/cs3org/reva/pkg/auth" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" ) func init() { diff --git a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go index b5cd899510..7a945eae2e 100644 --- a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go +++ b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go @@ -23,7 +23,7 @@ import ( "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/registry" "github.com/cs3org/reva/pkg/auth" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" ) func init() { diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index ab0bf42312..0fd6f8fe4e 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -36,7 +36,7 @@ import ( "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" diff --git a/internal/http/services/owncloud/ocdav/dav.go b/internal/http/services/owncloud/ocdav/dav.go index 65ba52e31a..07327337ca 100644 --- a/internal/http/services/owncloud/ocdav/dav.go +++ b/internal/http/services/owncloud/ocdav/dav.go @@ -30,7 +30,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" "google.golang.org/grpc/metadata" diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 0e93d56737..8b75f74574 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -31,7 +31,7 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/notification/notificationhelper" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index 51c5724358..e17dd1f2d5 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -41,7 +41,7 @@ import ( "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/share" diff --git a/internal/http/services/owncloud/ocdav/proppatch.go b/internal/http/services/owncloud/ocdav/proppatch.go index cfb0b1b513..3d0c91ee41 100644 --- a/internal/http/services/owncloud/ocdav/proppatch.go +++ b/internal/http/services/owncloud/ocdav/proppatch.go @@ -30,7 +30,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/pkg/errors" "github.com/rs/zerolog" ) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index bedcad8c14..837235a4fd 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -27,7 +27,7 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" ) const ( diff --git a/internal/http/services/owncloud/ocdav/trashbin.go b/internal/http/services/owncloud/ocdav/trashbin.go index 1bc2b5248b..3c24afbb1e 100644 --- a/internal/http/services/owncloud/ocdav/trashbin.go +++ b/internal/http/services/owncloud/ocdav/trashbin.go @@ -33,7 +33,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/utils" diff --git a/internal/http/services/owncloud/ocs/cache.go b/internal/http/services/owncloud/ocs/cache.go index bfb9b35d49..56cb588ba4 100644 --- a/internal/http/services/owncloud/ocs/cache.go +++ b/internal/http/services/owncloud/ocs/cache.go @@ -24,7 +24,7 @@ import ( "net/http/httptest" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc/metadata" ) diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go index 8041595938..96f6acf0b9 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go @@ -29,7 +29,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go index 312b55f3fb..e693150baf 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go @@ -31,7 +31,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 853b7771d2..b7d375af3d 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -47,7 +47,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/notification/notificationhelper" "github.com/cs3org/reva/pkg/notification/trigger" diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go index 57ab112490..17b44a2dd0 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go @@ -31,7 +31,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go b/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go index 34772315ed..c554be8a3c 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go @@ -23,7 +23,7 @@ import ( "strings" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/data" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/juliangruber/go-intersect" ) diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go b/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go index ba8bffc5c9..0ff8bf0e48 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/config" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go index 67638144cf..53fe196260 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go @@ -31,7 +31,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/go-chi/chi/v5" ) diff --git a/internal/http/services/reverseproxy/reverseproxy.go b/internal/http/services/reverseproxy/reverseproxy.go index 2c81b20815..0ae90746d4 100644 --- a/internal/http/services/reverseproxy/reverseproxy.go +++ b/internal/http/services/reverseproxy/reverseproxy.go @@ -26,7 +26,7 @@ import ( "net/url" "os" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" diff --git a/internal/http/services/sciencemesh/token.go b/internal/http/services/sciencemesh/token.go index 4c519c038d..54f97ea752 100644 --- a/internal/http/services/sciencemesh/token.go +++ b/internal/http/services/sciencemesh/token.go @@ -32,7 +32,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/smtpclient" "github.com/cs3org/reva/pkg/utils/list" diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 690711e14d..4e6e82912e 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -44,7 +44,7 @@ import ( "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/pkg/appauth/manager/json/json.go b/pkg/appauth/manager/json/json.go index 1ff1a667f9..650e1ba742 100644 --- a/pkg/appauth/manager/json/json.go +++ b/pkg/appauth/manager/json/json.go @@ -32,7 +32,7 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appauth" "github.com/cs3org/reva/pkg/appauth/manager/registry" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" diff --git a/pkg/appauth/manager/json/json_test.go b/pkg/appauth/manager/json/json_test.go index e20d1cd585..14b0b97c8f 100644 --- a/pkg/appauth/manager/json/json_test.go +++ b/pkg/appauth/manager/json/json_test.go @@ -32,7 +32,7 @@ import ( apppb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/gdexlab/go-render/render" "github.com/sethvargo/go-password/password" "golang.org/x/crypto/bcrypt" diff --git a/pkg/ctx/agentctx.go b/pkg/appctx/agentctx.go similarity index 99% rename from pkg/ctx/agentctx.go rename to pkg/appctx/agentctx.go index 82e6376587..c349136319 100644 --- a/pkg/ctx/agentctx.go +++ b/pkg/appctx/agentctx.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package ctx +package appctx import ( "context" diff --git a/pkg/ctx/agentctx_test.go b/pkg/appctx/agentctx_test.go similarity index 99% rename from pkg/ctx/agentctx_test.go rename to pkg/appctx/agentctx_test.go index ad4e0c165f..e99112b4b1 100644 --- a/pkg/ctx/agentctx_test.go +++ b/pkg/appctx/agentctx_test.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package ctx +package appctx import ( "context" diff --git a/pkg/appctx/appctx.go b/pkg/appctx/logger.go similarity index 100% rename from pkg/appctx/appctx.go rename to pkg/appctx/logger.go diff --git a/pkg/ctx/tokenctx.go b/pkg/appctx/tokenctx.go similarity index 99% rename from pkg/ctx/tokenctx.go rename to pkg/appctx/tokenctx.go index e1b431bd12..b84866645e 100644 --- a/pkg/ctx/tokenctx.go +++ b/pkg/appctx/tokenctx.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package ctx +package appctx import ( "context" diff --git a/pkg/ctx/userctx.go b/pkg/appctx/userctx.go similarity index 99% rename from pkg/ctx/userctx.go rename to pkg/appctx/userctx.go index 2cca7f5660..7ee6254065 100644 --- a/pkg/ctx/userctx.go +++ b/pkg/appctx/userctx.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package ctx +package appctx import ( "context" diff --git a/pkg/auth/manager/nextcloud/nextcloud_test.go b/pkg/auth/manager/nextcloud/nextcloud_test.go index b1998d4106..81bf6a82cb 100644 --- a/pkg/auth/manager/nextcloud/nextcloud_test.go +++ b/pkg/auth/manager/nextcloud/nextcloud_test.go @@ -28,7 +28,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/auth/manager/nextcloud" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/cbox/favorite/sql/sql.go b/pkg/cbox/favorite/sql/sql.go index 5e2ef9b238..8e74a8de30 100644 --- a/pkg/cbox/favorite/sql/sql.go +++ b/pkg/cbox/favorite/sql/sql.go @@ -26,7 +26,7 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/cbox/utils" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/storage/favorite" "github.com/cs3org/reva/pkg/storage/favorite/registry" "github.com/cs3org/reva/pkg/utils/cfg" diff --git a/pkg/cbox/preferences/sql/sql.go b/pkg/cbox/preferences/sql/sql.go index 8e0b2afbd5..c46469245e 100644 --- a/pkg/cbox/preferences/sql/sql.go +++ b/pkg/cbox/preferences/sql/sql.go @@ -23,7 +23,7 @@ import ( "database/sql" "fmt" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" diff --git a/pkg/cbox/share/sql/sql.go b/pkg/cbox/share/sql/sql.go index 0133ed8ddb..2671009932 100644 --- a/pkg/cbox/share/sql/sql.go +++ b/pkg/cbox/share/sql/sql.go @@ -33,7 +33,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" conversions "github.com/cs3org/reva/pkg/cbox/utils" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share" diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go index ec385dcef4..7fd7eac89c 100644 --- a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go +++ b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go @@ -25,7 +25,7 @@ import ( "github.com/Masterminds/sprig" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go index d4b518db40..984f35cdd4 100644 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -27,7 +27,7 @@ import ( "github.com/Masterminds/sprig" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" diff --git a/pkg/ctx/logger.go b/pkg/ctx/logger.go deleted file mode 100644 index a6907782b6..0000000000 --- a/pkg/ctx/logger.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package ctx - -import ( - "context" - - "github.com/cs3org/reva/pkg/trace" - "github.com/rs/zerolog" -) - -// WithLogger returns a context with an associated logger. -func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { - traceID := trace.Get(ctx) - sublog := l.With().Str(trace.Key, traceID).Logger() - return sublog.WithContext(ctx) -} - -// GetLogger returns the logger associated with the given context -// or a disabled logger in case no logger is stored inside the context. -func GetLogger(ctx context.Context) *zerolog.Logger { - return zerolog.Ctx(ctx) -} diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 1a47aa8240..2fe75f1eb7 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -33,7 +33,7 @@ import ( "time" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/acl" diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index 075c3a4292..3abfc75956 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -6,6 +6,7 @@ import ( "io" "net/http" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/trace" //"net/url" ) @@ -43,7 +44,7 @@ func (t injectTransport) RoundTrip(r *http.Request) (*http.Response, error) { tkn, ok := appctx.ContextGetToken(ctx) if ok { - httpReq.Header.Set(appctx.TokenHeader, tkn) + r.Header.Set(appctx.TokenHeader, tkn) } return t.rt.RoundTrip(r) diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go index 72533dbe2f..b2e5375df4 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go @@ -28,7 +28,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" masked_share "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/nextcloud" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index dbc9573fc3..526817862d 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -34,7 +34,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" diff --git a/pkg/preferences/memory/memory.go b/pkg/preferences/memory/memory.go index 10ff617fa3..ed069932f3 100644 --- a/pkg/preferences/memory/memory.go +++ b/pkg/preferences/memory/memory.go @@ -22,7 +22,7 @@ import ( "context" "sync" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" diff --git a/pkg/rhttp/client.go b/pkg/rhttp/client.go index 2fc6662c67..d2d844c4df 100644 --- a/pkg/rhttp/client.go +++ b/pkg/rhttp/client.go @@ -24,7 +24,7 @@ import ( "io" "net/http" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/pkg/errors" "go.opencensus.io/plugin/ochttp" ) diff --git a/pkg/sdk/common/net/net.go b/pkg/sdk/common/net/net.go index 2e73cf2d3d..7db9c5056b 100644 --- a/pkg/sdk/common/net/net.go +++ b/pkg/sdk/common/net/net.go @@ -20,7 +20,7 @@ package net import ( "github.com/cs3org/reva/internal/http/services/datagateway" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" ) type ctxKey int diff --git a/pkg/share/cache/warmup/cbox/cbox.go b/pkg/share/cache/warmup/cbox/cbox.go index 0ff270565e..cc8b55e2ef 100644 --- a/pkg/share/cache/warmup/cbox/cbox.go +++ b/pkg/share/cache/warmup/cbox/cbox.go @@ -27,7 +27,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share/cache" "github.com/cs3org/reva/pkg/share/cache/warmup/registry" diff --git a/pkg/share/manager/json/json.go b/pkg/share/manager/json/json.go index a453b552cd..4646fbb8f6 100644 --- a/pkg/share/manager/json/json.go +++ b/pkg/share/manager/json/json.go @@ -29,7 +29,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" diff --git a/pkg/share/manager/memory/memory.go b/pkg/share/manager/memory/memory.go index 2646744a74..93094b759f 100644 --- a/pkg/share/manager/memory/memory.go +++ b/pkg/share/manager/memory/memory.go @@ -29,7 +29,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" diff --git a/pkg/share/manager/sql/sql.go b/pkg/share/manager/sql/sql.go index 984e0f6425..8e22fac2b6 100644 --- a/pkg/share/manager/sql/sql.go +++ b/pkg/share/manager/sql/sql.go @@ -30,7 +30,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" diff --git a/pkg/share/manager/sql/sql_test.go b/pkg/share/manager/sql/sql_test.go index bd70c9a358..2401c60387 100644 --- a/pkg/share/manager/sql/sql_test.go +++ b/pkg/share/manager/sql/sql_test.go @@ -26,7 +26,7 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ruser "github.com/cs3org/reva/pkg/ctx" + ruser "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/share" sqlmanager "github.com/cs3org/reva/pkg/share/manager/sql" mocks "github.com/cs3org/reva/pkg/share/manager/sql/mocks" diff --git a/pkg/storage/favorite/memory/memory_test.go b/pkg/storage/favorite/memory/memory_test.go index ac2f766497..d9869c6d99 100644 --- a/pkg/storage/favorite/memory/memory_test.go +++ b/pkg/storage/favorite/memory/memory_test.go @@ -24,7 +24,7 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" ) type environment struct { diff --git a/pkg/storage/fs/cback/cback.go b/pkg/storage/fs/cback/cback.go index a9ceb153e4..874f4fe8c0 100644 --- a/pkg/storage/fs/cback/cback.go +++ b/pkg/storage/fs/cback/cback.go @@ -29,7 +29,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" v1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/rhttp" diff --git a/pkg/storage/fs/cephfs/upload.go b/pkg/storage/fs/cephfs/upload.go index 5c6d929e1f..0f340e7515 100644 --- a/pkg/storage/fs/cephfs/upload.go +++ b/pkg/storage/fs/cephfs/upload.go @@ -33,7 +33,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctx2 "github.com/cs3org/reva/pkg/ctx" + ctx2 "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/utils" "github.com/google/uuid" diff --git a/pkg/storage/fs/cephfs/user.go b/pkg/storage/fs/cephfs/user.go index 71e5f436ac..aa5a2723b2 100644 --- a/pkg/storage/fs/cephfs/user.go +++ b/pkg/storage/fs/cephfs/user.go @@ -35,7 +35,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctx2 "github.com/cs3org/reva/pkg/ctx" + ctx2 "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/pkg/errors" diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 58c6a65508..3382568b2f 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -33,7 +33,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index f388e8da27..b9053830a6 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -31,7 +31,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/storage/fs/nextcloud" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" . "github.com/onsi/ginkgo" diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index eb23d12257..4d6f836b1e 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -38,7 +38,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/mime" diff --git a/pkg/storage/fs/owncloud/upload.go b/pkg/storage/fs/owncloud/upload.go index 04df416185..c2ad6fa22d 100644 --- a/pkg/storage/fs/owncloud/upload.go +++ b/pkg/storage/fs/owncloud/upload.go @@ -34,7 +34,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/storage/utils/chunking" diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 03d410a90c..e45eb6a5af 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -43,7 +43,7 @@ import ( "github.com/cs3org/reva/internal/grpc/services/storageprovider" conversions "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/mime" diff --git a/pkg/storage/fs/owncloudsql/upload.go b/pkg/storage/fs/owncloudsql/upload.go index 5bcfa25c6d..05f5553d68 100644 --- a/pkg/storage/fs/owncloudsql/upload.go +++ b/pkg/storage/fs/owncloudsql/upload.go @@ -31,7 +31,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" conversions "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/mime" diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index cef0ec69b1..244a1ae699 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -26,7 +26,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" diff --git a/pkg/storage/registry/static/static_test.go b/pkg/storage/registry/static/static_test.go index 2f6639511e..5476fdea75 100644 --- a/pkg/storage/registry/static/static_test.go +++ b/pkg/storage/registry/static/static_test.go @@ -24,7 +24,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/storage/registry/static" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 7e9201e655..6ad33f0c7f 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -43,7 +43,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/eosclient/eosbinary" "github.com/cs3org/reva/pkg/eosclient/eosgrpc" diff --git a/pkg/storage/utils/eosfs/eosfs_test.go b/pkg/storage/utils/eosfs/eosfs_test.go index 9abd13a266..c2ef3a8947 100644 --- a/pkg/storage/utils/eosfs/eosfs_test.go +++ b/pkg/storage/utils/eosfs/eosfs_test.go @@ -30,7 +30,7 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" "github.com/gdexlab/go-render/render" "github.com/thanhpk/randstr" diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index 22dcaecaeb..b204b0fe47 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -35,7 +35,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" diff --git a/pkg/storage/utils/localfs/upload.go b/pkg/storage/utils/localfs/upload.go index e2021250b6..0c2727a537 100644 --- a/pkg/storage/utils/localfs/upload.go +++ b/pkg/storage/utils/localfs/upload.go @@ -28,7 +28,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/utils" diff --git a/pkg/user/manager/nextcloud/nextcloud.go b/pkg/user/manager/nextcloud/nextcloud.go index cd0422b22f..98e4cab03a 100644 --- a/pkg/user/manager/nextcloud/nextcloud.go +++ b/pkg/user/manager/nextcloud/nextcloud.go @@ -29,7 +29,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" diff --git a/pkg/user/manager/nextcloud/nextcloud_test.go b/pkg/user/manager/nextcloud/nextcloud_test.go index a9114abd68..a33821d164 100644 --- a/pkg/user/manager/nextcloud/nextcloud_test.go +++ b/pkg/user/manager/nextcloud/nextcloud_test.go @@ -24,7 +24,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/pkg/user/manager/nextcloud" "github.com/cs3org/reva/tests/helpers" diff --git a/tests/integration/grpc/ocm_invitation_test.go b/tests/integration/grpc/ocm_invitation_test.go index c4addc6c79..1e3626d602 100644 --- a/tests/integration/grpc/ocm_invitation_test.go +++ b/tests/integration/grpc/ocm_invitation_test.go @@ -33,7 +33,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 5e9eda0818..89087dde04 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -24,7 +24,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" . "github.com/onsi/ginkgo" From 21e242341bd10ce81a0f5b9c8b78d7df8e525cf2 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 27 Sep 2023 14:20:25 +0200 Subject: [PATCH 10/25] refactor rhttp to httpclient --- cmd/reva/download.go | 3 +- cmd/reva/main.go | 13 +- cmd/reva/upload.go | 7 +- .../services/appregistry/appregistry_test.go | 18 +-- internal/grpc/services/pingpong/pingpong.go | 4 +- .../http/services/appprovider/appprovider.go | 12 +- internal/http/services/archiver/handler.go | 8 +- .../http/services/datagateway/datagateway.go | 23 ++-- internal/http/services/overleaf/overleaf.go | 7 +- internal/http/services/owncloud/ocdav/copy.go | 9 +- internal/http/services/owncloud/ocdav/get.go | 3 +- .../http/services/owncloud/ocdav/ocdav.go | 12 +- internal/http/services/owncloud/ocdav/put.go | 3 +- internal/http/services/owncloud/ocdav/tpc.go | 5 +- internal/http/services/owncloud/ocdav/tus.go | 3 +- .../http/services/owncloud/ocdav/versions.go | 3 +- pkg/app/provider/wopi/wopi.go | 31 +++-- pkg/auth/manager/oidc/oidc.go | 18 +-- pkg/cbox/utils/tokenmanagement.go | 13 +- pkg/datatx/manager/rclone/rclone.go | 8 +- pkg/httpclient/httpclient.go | 114 ++++++++++++++---- pkg/ocm/client/client.go | 14 ++- pkg/ocm/provider/authorizer/mentix/mentix.go | 14 ++- pkg/ocm/share/sender/sender.go | 6 +- pkg/ocm/storage/outcoming/ocm.go | 5 +- pkg/rhttp/client.go | 69 ----------- pkg/rhttp/option.go | 74 ------------ pkg/storage/fs/cback/cback.go | 7 +- pkg/storage/utils/downloader/downloader.go | 10 +- tests/helpers/helpers.go | 7 +- 30 files changed, 234 insertions(+), 289 deletions(-) delete mode 100644 pkg/rhttp/client.go delete mode 100644 pkg/rhttp/option.go diff --git a/cmd/reva/download.go b/cmd/reva/download.go index 5fb2747e63..99eb09ccd0 100644 --- a/cmd/reva/download.go +++ b/cmd/reva/download.go @@ -31,7 +31,6 @@ import ( "github.com/cs3org/reva/internal/http/services/datagateway" ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" "github.com/studio-b12/gowebdav" @@ -95,7 +94,7 @@ func downloadCommand() *command { dataServerURL := p.DownloadEndpoint // TODO(labkode): do a protocol switch - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, dataServerURL, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dataServerURL, nil) if err != nil { return err } diff --git a/cmd/reva/main.go b/cmd/reva/main.go index 4a5e06d3ac..c7c6558dbc 100644 --- a/cmd/reva/main.go +++ b/cmd/reva/main.go @@ -19,6 +19,7 @@ package main import ( + "crypto/tls" "flag" "fmt" "net/http" @@ -27,7 +28,7 @@ import ( "time" "github.com/c-bata/go-prompt" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" ) var ( @@ -40,7 +41,7 @@ var ( gitCommit, buildDate, version, goVersion string - client *http.Client + client *httpclient.Client commands = []*command{ versionCommand(), @@ -125,9 +126,11 @@ func main() { } } - client = rhttp.GetHTTPClient( - rhttp.Insecure(insecuredatagateway), - rhttp.Timeout(time.Duration(timeout*int64(time.Hour))), + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecuredatagateway}} + + client = httpclient.New( + httpclient.RoundTripper(tr), + httpclient.Timeout(time.Duration(timeout*int64(time.Hour))), ) generateMainUsage() diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index a3056ae51f..4ea46f6d09 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -35,10 +35,9 @@ import ( // TODO(labkode): this should not come from this package. "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/datagateway" - "github.com/cs3org/reva/pkg/crypto" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/crypto" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" "github.com/eventials/go-tus" "github.com/eventials/go-tus/memorystore" @@ -147,7 +146,7 @@ func uploadCommand() *command { dataServerURL := p.UploadEndpoint if *protocolFlag == "simple" { - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, dataServerURL, fd) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, dataServerURL, fd) if err != nil { return err } @@ -170,7 +169,7 @@ func uploadCommand() *command { // create the tus client. c := tus.DefaultConfig() c.Resume = true - c.HttpClient = client + c.HttpClient = client.GetNativeHTTP() c.Store, err = memorystore.NewMemoryStore() if err != nil { return err diff --git a/internal/grpc/services/appregistry/appregistry_test.go b/internal/grpc/services/appregistry/appregistry_test.go index f56dad5997..9a265f1e81 100644 --- a/internal/grpc/services/appregistry/appregistry_test.go +++ b/internal/grpc/services/appregistry/appregistry_test.go @@ -76,7 +76,7 @@ func Test_ListAppProviders(t *testing.T) { want: ®istrypb.ListAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "", }, Providers: []*registrypb.ProviderInfo{ @@ -98,7 +98,7 @@ func Test_ListAppProviders(t *testing.T) { want: ®istrypb.ListAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, - Trace: "00000000000000000000000000000000", + Trace: "", }, Providers: []*registrypb.ProviderInfo{}, }, @@ -112,7 +112,7 @@ func Test_ListAppProviders(t *testing.T) { want: ®istrypb.ListAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "", }, Providers: []*registrypb.ProviderInfo{}, @@ -218,7 +218,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "", }, Providers: []*registrypb.ProviderInfo{ @@ -235,7 +235,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "", }, Providers: []*registrypb.ProviderInfo{ @@ -252,7 +252,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 15, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "error looking for the app provider", }, Providers: nil, @@ -264,7 +264,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 15, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "error looking for the app provider", }, Providers: nil, @@ -276,7 +276,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 15, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "error looking for the app provider", }, Providers: nil, @@ -288,7 +288,7 @@ func Test_GetAppProviders(t *testing.T) { want: ®istrypb.GetAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 15, - Trace: "00000000000000000000000000000000", + Trace: "", Message: "error looking for the app provider", }, Providers: nil, diff --git a/internal/grpc/services/pingpong/pingpong.go b/internal/grpc/services/pingpong/pingpong.go index 284332de15..564a3daff4 100644 --- a/internal/grpc/services/pingpong/pingpong.go +++ b/internal/grpc/services/pingpong/pingpong.go @@ -81,8 +81,8 @@ func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingRe TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } - client := httpclient.NewWithRoundTripper(tr) - req, err := httpclient.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) + client := httpclient.New(httpclient.RoundTripper(tr)) + req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) if err != nil { log.Error().Err(err).Msg("error creating http request") return nil, err diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 14289f9006..151b8d179e 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -20,6 +20,7 @@ package appprovider import ( "context" + "crypto/tls" "encoding/json" "net/http" "path" @@ -32,9 +33,9 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" @@ -234,17 +235,16 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { ep, token = p.UploadEndpoint, p.Token } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, ep, nil) if err != nil { writeError(w, r, appErrorServerError, "failed to create the file", err) return } httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := rhttp.GetHTTPClient( - rhttp.Context(ctx), - rhttp.Insecure(s.conf.Insecure), - ).Do(httpReq) + + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: s.conf.Insecure}} + httpRes, err := httpclient.New(httpclient.RoundTripper(tr)).Do(httpReq) if err != nil { writeError(w, r, appErrorServerError, "failed to create the file", err) return diff --git a/internal/http/services/archiver/handler.go b/internal/http/services/archiver/handler.go index 2f43e2b1e1..a8a559563d 100644 --- a/internal/http/services/archiver/handler.go +++ b/internal/http/services/archiver/handler.go @@ -20,6 +20,7 @@ package archiver import ( "context" + "crypto/tls" "errors" "fmt" "net/http" @@ -34,8 +35,8 @@ import ( "github.com/cs3org/reva/internal/http/services/archiver/manager" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage/utils/downloader" @@ -93,10 +94,13 @@ func New(ctx context.Context, conf map[string]interface{}) (global.Service, erro allowedFolderRegex = append(allowedFolderRegex, regex) } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}} + hc := httpclient.New(httpclient.RoundTripper(tr), httpclient.Timeout(time.Duration(c.Timeout*int64(time.Second)))) + return &svc{ config: &c, gtwClient: gtw, - downloader: downloader.NewDownloader(gtw, rhttp.Insecure(c.Insecure), rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second)))), + downloader: downloader.NewDownloader(gtw, hc), walker: walker.NewWalker(gtw), allowedFolders: allowedFolderRegex, }, nil diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index e1b80d8b21..f0724aa2f2 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -20,6 +20,7 @@ package datagateway import ( "context" + "crypto/tls" "io" "net/http" "net/url" @@ -29,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils/cfg" @@ -72,7 +73,7 @@ func (c *config) ApplyDefaults() { type svc struct { conf *config handler http.Handler - client *http.Client + client *httpclient.Client } // New returns a new datagateway. @@ -82,11 +83,15 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) return nil, err } + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}, + } + s := &svc{ conf: &c, - client: rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), - rhttp.Insecure(c.Insecure), + client: httpclient.New( + httpclient.Timeout(time.Duration(c.Timeout*int64(time.Second))), + httpclient.RoundTripper(tr), ), } s.setHandler() @@ -181,7 +186,7 @@ func (s *svc) doHead(w http.ResponseWriter, r *http.Request) { log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, http.MethodHead, claims.Target, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodHead, claims.Target, nil) if err != nil { log.Error().Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) @@ -233,7 +238,7 @@ func (s *svc) doGet(w http.ResponseWriter, r *http.Request) { log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, claims.Target, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, claims.Target, nil) if err != nil { log.Error().Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) @@ -310,7 +315,7 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, target, r.Body) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, target, r.Body) if err != nil { log.Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) @@ -369,7 +374,7 @@ func (s *svc) doPatch(w http.ResponseWriter, r *http.Request) { log.Debug().Str("target", claims.Target).Msg("sending request to internal data server") httpClient := s.client - httpReq, err := rhttp.NewRequest(ctx, http.MethodPatch, target, r.Body) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPatch, target, r.Body) if err != nil { log.Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 0fd6f8fe4e..b4a668dc9b 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -35,10 +35,9 @@ import ( storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/token/manager/jwt" @@ -211,7 +210,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { } // Setting up archiver request - archHTTPReq, err := rhttp.NewRequest(ctx, http.MethodGet, s.conf.ArchiverURL, nil) + archHTTPReq, err := http.NewRequestWithContext(ctx, http.MethodGet, s.conf.ArchiverURL, nil) if err != nil { reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting up http request", nil) return @@ -227,7 +226,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { // Setting up Overleaf request appURL := s.conf.appURL + "/docs" - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, appURL, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, appURL, nil) if err != nil { reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting up http request", nil) return diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index 3dfdd9a699..e3bf05fda5 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -32,7 +32,6 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/utils" "github.com/rs/zerolog" @@ -233,7 +232,7 @@ func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClie // 3. do download - httpDownloadReq, err := rhttp.NewRequest(ctx, http.MethodGet, downloadEP, nil) + httpDownloadReq, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadEP, nil) if err != nil { return err } @@ -250,7 +249,7 @@ func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClie // 4. do upload - httpUploadReq, err := rhttp.NewRequest(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) + httpUploadReq, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) if err != nil { return err } @@ -447,7 +446,7 @@ func (s *svc) executeSpacesCopy(ctx context.Context, w http.ResponseWriter, clie } // 3. do download - httpDownloadReq, err := rhttp.NewRequest(ctx, http.MethodGet, downloadEP, nil) + httpDownloadReq, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadEP, nil) if err != nil { return err } @@ -466,7 +465,7 @@ func (s *svc) executeSpacesCopy(ctx context.Context, w http.ResponseWriter, clie // 4. do upload - httpUploadReq, err := rhttp.NewRequest(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) + httpUploadReq, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) if err != nil { return err } diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index eedb923fbb..9343d1c85d 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -33,7 +33,6 @@ import ( "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" @@ -91,7 +90,7 @@ func (s *svc) handleGet(ctx context.Context, w http.ResponseWriter, r *http.Requ } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, ep, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, ep, nil) if err != nil { log.Error().Err(err).Msg("error creating http request") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 8b75f74574..8aa9ec767a 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -20,6 +20,7 @@ package ocdav import ( "context" + "crypto/tls" "fmt" "net/http" "net/url" @@ -33,9 +34,9 @@ import ( "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/notification/notificationhelper" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" @@ -132,7 +133,7 @@ type svc struct { webDavHandler *WebDavHandler davHandler *DavHandler favoritesManager favorite.Manager - client *http.Client + client *httpclient.Client notificationHelper *notificationhelper.NotificationHelper } @@ -156,13 +157,14 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) } log := appctx.GetLogger(ctx) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}} s := &svc{ c: &c, webDavHandler: new(WebDavHandler), davHandler: new(DavHandler), - client: rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), - rhttp.Insecure(c.Insecure), + client: httpclient.New( + httpclient.Timeout(time.Duration(c.Timeout*int64(time.Second))), + httpclient.RoundTripper(tr), ), favoritesManager: fm, notificationHelper: notificationhelper.New("ocdav", c.Notifications, log), diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 9f18985259..8f5f9c8ad3 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -36,7 +36,6 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/notification/trigger" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" @@ -252,7 +251,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, r.Body) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, ep, r.Body) if err != nil { w.WriteHeader(http.StatusInternalServerError) return diff --git a/internal/http/services/owncloud/ocdav/tpc.go b/internal/http/services/owncloud/ocdav/tpc.go index f6922abf6e..8e1007e4c3 100644 --- a/internal/http/services/owncloud/ocdav/tpc.go +++ b/internal/http/services/owncloud/ocdav/tpc.go @@ -36,7 +36,6 @@ import ( "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" ) const ( @@ -240,7 +239,7 @@ func (s *svc) performHTTPPull(ctx context.Context, client gateway.GatewayAPIClie tempReader := io.TeeReader(httpDownloadRes.Body, &wc) // do Upload - httpUploadReq, err := rhttp.NewRequest(ctx, http.MethodPut, uploadEP, tempReader) + httpUploadReq, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadEP, tempReader) if err != nil { w.WriteHeader(http.StatusInternalServerError) return err @@ -362,7 +361,7 @@ func (s *svc) performHTTPPush(ctx context.Context, client gateway.GatewayAPIClie } // do download - httpDownloadReq, err := rhttp.NewRequest(ctx, http.MethodGet, downloadEP, nil) + httpDownloadReq, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadEP, nil) if err != nil { w.WriteHeader(http.StatusInternalServerError) return err diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index b3a8fa8402..e533af289b 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -33,7 +33,6 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" @@ -221,7 +220,7 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http. var httpRes *http.Response - httpReq, err := rhttp.NewRequest(ctx, http.MethodPatch, ep, r.Body) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPatch, ep, r.Body) if err != nil { log.Debug().Err(err).Msg("wrong request") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/http/services/owncloud/ocdav/versions.go b/internal/http/services/owncloud/ocdav/versions.go index 463b4dc8be..c8953444be 100644 --- a/internal/http/services/owncloud/ocdav/versions.go +++ b/internal/http/services/owncloud/ocdav/versions.go @@ -30,6 +30,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/storage/utils/downloader" "github.com/cs3org/reva/pkg/utils/resourceid" @@ -246,7 +247,7 @@ func (h *VersionsHandler) doDownload(w http.ResponseWriter, r *http.Request, s * w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fname)) w.Header().Set("Content-Transfer-Encoding", "binary") - down := downloader.NewDownloader(client) + down := downloader.NewDownloader(client, httpclient.New()) d, err := down.Download(ctx, resStat.Info.Path, key) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 4e6e82912e..e5407d32c3 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -21,6 +21,7 @@ package wopi import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -43,12 +44,12 @@ import ( "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" @@ -102,7 +103,7 @@ func (c *config) ApplyDefaults() { type wopiProvider struct { conf *config - wopiClient *http.Client + wopiClient *httpclient.Client appURLs map[string]map[string]string // map[viewMode]map[extension]appURL } @@ -119,14 +120,18 @@ func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { return nil, err } - wopiClient := rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(10*int64(time.Second))), - rhttp.Insecure(c.InsecureConnections), - ) - wopiClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.InsecureConnections}} + + cr := func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } + wopiClient := httpclient.New( + httpclient.Timeout(time.Duration(10*int64(time.Second))), + httpclient.RoundTripper(tr), + httpclient.CheckRedirect(cr), + ) + return &wopiProvider{ conf: &c, wopiClient: wopiClient, @@ -144,7 +149,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc } wopiurl.Path = path.Join(wopiurl.Path, "/wopi/iop/openinapp") - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, wopiurl.String(), nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, wopiurl.String(), nil) if err != nil { return nil, err } @@ -365,9 +370,11 @@ func (p *wopiProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.Pro func getAppURLs(c *config) (map[string]map[string]string, error) { // Initialize WOPI URLs by discovery - httpcl := rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(5*int64(time.Second))), - rhttp.Insecure(c.InsecureConnections), + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.InsecureConnections}} + + httpcl := httpclient.New( + httpclient.Timeout(time.Duration(5*int64(time.Second))), + httpclient.RoundTripper(tr), ) appurl, err := url.Parse(c.AppIntURL) diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index 4df459d9af..93e3c83806 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -22,8 +22,10 @@ package oidc import ( "context" + "crypto/tls" "encoding/json" "fmt" + "net/http" "os" "strings" "time" @@ -37,9 +39,9 @@ import ( "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/golang-jwt/jwt" @@ -294,14 +296,16 @@ func (am *mgr) Authenticate(ctx context.Context, _, clientSecret string) (*user. } func (am *mgr) getOAuthCtx(ctx context.Context) context.Context { + tr := &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{InsecureSkipVerify: am.c.Insecure}, + } + // Sometimes for testing we need to skip the TLS check, that's why we need a // custom HTTP client. - customHTTPClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - rhttp.Timeout(time.Second*10), - rhttp.Insecure(am.c.Insecure), - // Fixes connection fd leak which might be caused by provider-caching - rhttp.DisableKeepAlive(true), + customHTTPClient := httpclient.New( + httpclient.Timeout(time.Second*10), + httpclient.RoundTripper(tr), ) ctx = context.WithValue(ctx, oauth2.HTTPClient, customHTTPClient) return ctx diff --git a/pkg/cbox/utils/tokenmanagement.go b/pkg/cbox/utils/tokenmanagement.go index 6eea766d92..0dbc704113 100644 --- a/pkg/cbox/utils/tokenmanagement.go +++ b/pkg/cbox/utils/tokenmanagement.go @@ -20,6 +20,7 @@ package utils import ( "context" + "crypto/tls" "encoding/json" "errors" "io" @@ -29,7 +30,7 @@ import ( "sync" "time" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" "github.com/mitchellh/mapstructure" ) @@ -37,7 +38,7 @@ import ( type APITokenManager struct { oidcToken OIDCToken conf *config - client *http.Client + client *httpclient.Client } // OIDCToken stores the OIDC token used to authenticate requests to the REST API service. @@ -63,11 +64,13 @@ func InitAPITokenManager(conf map[string]interface{}) (*APITokenManager, error) return nil, err } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}} + return &APITokenManager{ conf: c, - client: rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), - rhttp.Insecure(c.Insecure), + client: httpclient.New( + httpclient.Timeout(time.Duration(c.Timeout*int64(time.Second))), + httpclient.RoundTripper(tr), )}, nil } diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index ba97088096..e99c8ea70f 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -21,6 +21,7 @@ package rclone import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "net/http" @@ -37,7 +38,7 @@ import ( repoRegistry "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository/registry" registry "github.com/cs3org/reva/pkg/datatx/manager/registry" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -72,7 +73,7 @@ type config struct { type rclone struct { config *config - client *http.Client + client *httpclient.Client storage repository.Repository } @@ -109,7 +110,8 @@ func New(ctx context.Context, m map[string]interface{}) (txdriver.Manager, error } c.init(m) - client := rhttp.GetHTTPClient(rhttp.Insecure(c.Insecure)) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}} + client := httpclient.New(httpclient.RoundTripper(tr)) storage, err := getStorageManager(ctx, c) if err != nil { diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index 3abfc75956..130e8bd2be 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -2,32 +2,104 @@ package httpclient import ( //"io" - "context" - "io" + + "errors" "net/http" + "time" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/trace" //"net/url" ) -// New creates an http.Client with a custom round tripper that adds tracing headers -// This client must be used in the reva codebase and usage of standard http.Client -func New() *http.Client { - t := &injectTransport{rt: http.DefaultTransport} - client := http.Client{ - Transport: t, +// TODO(labkode): harden it. +// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779 +func New(opts ...Option) *Client { + options := newOptions(opts...) + + var tr http.RoundTripper + if options.RoundTripper == nil { + tr = &injectTransport{rt: http.DefaultTransport} + } else { + tr = &injectTransport{rt: options.RoundTripper} + } + + httpClient := &http.Client{ + Timeout: options.Timeout, + Transport: tr, + } + + return &Client{c: httpClient} +} + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Jar http.CookieJar + CheckRedirect func(req *http.Request, via []*http.Request) error + Timeout time.Duration + RoundTripper http.RoundTripper +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) } - return &client + + return opt } -// NewWithRoundTripper works as New but wraps the rt argument with the custom round tripper -func NewWithRoundTripper(rt http.RoundTripper) *http.Client { - t := &injectTransport{rt: rt} - client := http.Client{ - Transport: t, +// Timeout provides a function to set the timeout option. +func Timeout(t time.Duration) Option { + return func(o *Options) { + o.Timeout = t } - return &client +} + +// RoundTripper provides a function to set a custom RoundTripper. +func RoundTripper(rt http.RoundTripper) Option { + return func(o *Options) { + o.RoundTripper = rt + } +} + +// CheckRedirect provides a function to set a custom CheckRedirect. +func CheckRedirect(cr func(req *http.Request, via []*http.Request) error) Option { + return func(o *Options) { + o.CheckRedirect = cr + } + +} + +// Jar provides a function to set a custom CookieJar. +func Jar(j http.CookieJar) Option { + return func(o *Options) { + o.Jar = j + } + +} + +// Client wraps a http.Client but only exposes the Do method +// to force consumers to always create a request with http.NewRequestWithContext() +type Client struct { + c *http.Client +} + +func (c *Client) Do(r *http.Request) (*http.Response, error) { + // bail out early if context is not set + if r.Context() == nil { + return nil, errors.New("error: request must have a context") + } + return c.c.Do(r) +} + +func (c *Client) GetNativeHTTP() *http.Client { + return c.c } type injectTransport struct { @@ -40,7 +112,7 @@ func (t injectTransport) RoundTrip(r *http.Request) (*http.Response, error) { traceID := trace.Get(ctx) - r.Header.Add("X-Trace-ID", traceID) + r.Header.Set("X-Trace-ID", traceID) tkn, ok := appctx.ContextGetToken(ctx) if ok { @@ -49,13 +121,3 @@ func (t injectTransport) RoundTrip(r *http.Request) (*http.Response, error) { return t.rt.RoundTrip(r) } - -// NewRequest creates an HTTP request that sets tracing headers -func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) { - httpReq, err := http.NewRequestWithContext(ctx, method, url, body) - if err != nil { - return nil, err - } - - return httpReq, nil -} diff --git a/pkg/ocm/client/client.go b/pkg/ocm/client/client.go index 651129bf5b..5c735678e7 100644 --- a/pkg/ocm/client/client.go +++ b/pkg/ocm/client/client.go @@ -21,6 +21,7 @@ package client import ( "bytes" "context" + "crypto/tls" "encoding/json" "io" "net/http" @@ -30,7 +31,7 @@ import ( "github.com/cs3org/reva/internal/http/services/ocmd" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" "github.com/pkg/errors" ) @@ -56,7 +57,7 @@ var ErrInvalidParameters = errors.New("invalid parameters") // OCMClient is the client for an OCM provider. type OCMClient struct { - client *http.Client + client *httpclient.Client } // Config is the configuration to be used for the OCMClient. @@ -67,10 +68,13 @@ type Config struct { // New returns a new OCMClient. func New(c *Config) *OCMClient { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}, + } return &OCMClient{ - client: rhttp.GetHTTPClient( - rhttp.Timeout(c.Timeout), - rhttp.Insecure(c.Insecure), + client: httpclient.New( + httpclient.Timeout(c.Timeout), + httpclient.RoundTripper(tr), ), } } diff --git a/pkg/ocm/provider/authorizer/mentix/mentix.go b/pkg/ocm/provider/authorizer/mentix/mentix.go index 88347270f5..30fb4c473a 100644 --- a/pkg/ocm/provider/authorizer/mentix/mentix.go +++ b/pkg/ocm/provider/authorizer/mentix/mentix.go @@ -20,6 +20,7 @@ package mentix import ( "context" + "crypto/tls" "encoding/json" "fmt" "net" @@ -31,9 +32,9 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/ocm/provider" "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -45,7 +46,7 @@ func init() { // Client is a Mentix API client. type Client struct { BaseURL string - HTTPClient *http.Client + HTTPClient *httpclient.Client } // New returns a new authorizer object. @@ -55,12 +56,13 @@ func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, er return nil, err } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}} + client := &Client{ BaseURL: c.URL, - HTTPClient: rhttp.GetHTTPClient( - rhttp.Context(context.Background()), - rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), - rhttp.Insecure(c.Insecure), + HTTPClient: httpclient.New( + httpclient.Timeout(time.Duration(c.Timeout*int64(time.Second))), + httpclient.RoundTripper(tr), ), } diff --git a/pkg/ocm/share/sender/sender.go b/pkg/ocm/share/sender/sender.go index 7cdb6222d5..4cd8e510b8 100644 --- a/pkg/ocm/share/sender/sender.go +++ b/pkg/ocm/share/sender/sender.go @@ -31,7 +31,7 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" "github.com/pkg/errors" ) @@ -73,8 +73,8 @@ func Send(ctx context.Context, requestBodyMap map[string]interface{}, pi *ocmpro return errors.Wrap(err, "sender: error framing post request") } req.Header.Set("Content-Type", "application/json") - client := rhttp.GetHTTPClient( - rhttp.Timeout(5 * time.Second), + client := httpclient.New( + httpclient.Timeout(5 * time.Second), ) resp, err := client.Do(req) diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 526817862d..6397f8f398 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -37,7 +37,6 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" @@ -410,7 +409,7 @@ func (d *driver) Upload(ctx context.Context, ref *provider.Reference, content io return errtypes.InternalError("simple upload not supported") } - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, endpoint, content) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint, content) if err != nil { return errors.Wrap(err, "error creating new request") } @@ -465,7 +464,7 @@ func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.Read return errtypes.InternalError("simple download not supported") } - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, endpoint, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return err } diff --git a/pkg/rhttp/client.go b/pkg/rhttp/client.go deleted file mode 100644 index d2d844c4df..0000000000 --- a/pkg/rhttp/client.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package rhttp - -import ( - "context" - "crypto/tls" - "io" - "net/http" - - ctxpkg "github.com/cs3org/reva/pkg/appctx" - "github.com/pkg/errors" - "go.opencensus.io/plugin/ochttp" -) - -// GetHTTPClient returns an http client with open census tracing support. -// TODO(labkode): harden it. -// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779 -func GetHTTPClient(opts ...Option) *http.Client { - options := newOptions(opts...) - - tr := http.DefaultTransport.(*http.Transport).Clone() - tr.DisableKeepAlives = options.DisableKeepAlive - tr.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: options.Insecure, - } - - httpClient := &http.Client{ - Timeout: options.Timeout, - Transport: &ochttp.Transport{ - Base: tr, - }, - } - - return httpClient -} - -// NewRequest creates an HTTP request that sets the token if it is passed in ctx. -func NewRequest(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) { - httpReq, err := http.NewRequest(method, url, body) - if err != nil { - return nil, errors.Wrap(err, "utils: error creating request") - } - - // TODO(labkode): make header / auth configurable - tkn, ok := ctxpkg.ContextGetToken(ctx) - if ok { - httpReq.Header.Set(ctxpkg.TokenHeader, tkn) - } - - httpReq = httpReq.WithContext(ctx) - return httpReq, nil -} diff --git a/pkg/rhttp/option.go b/pkg/rhttp/option.go deleted file mode 100644 index c5d41ff8b9..0000000000 --- a/pkg/rhttp/option.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package rhttp - -import ( - "context" - "time" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - Context context.Context - Timeout time.Duration - Insecure bool - DisableKeepAlive bool -} - -// newOptions initializes the available default options. -func newOptions(opts ...Option) Options { - opt := Options{} - - for _, o := range opts { - o(&opt) - } - - return opt -} - -// Context provides a function to set the context option. -func Context(val context.Context) Option { - return func(o *Options) { - o.Context = val - } -} - -// Insecure provides a function to set the insecure option. -func Insecure(insecure bool) Option { - return func(o *Options) { - o.Insecure = insecure - } -} - -// Timeout provides a function to set the timeout option. -func Timeout(t time.Duration) Option { - return func(o *Options) { - o.Timeout = t - } -} - -// DisableKeepAlive provides a function to set the disablee keep alive option. -func DisableKeepAlive(disable bool) Option { - return func(o *Options) { - o.DisableKeepAlive = disable - } -} diff --git a/pkg/storage/fs/cback/cback.go b/pkg/storage/fs/cback/cback.go index 874f4fe8c0..16c608bf8e 100644 --- a/pkg/storage/fs/cback/cback.go +++ b/pkg/storage/fs/cback/cback.go @@ -31,8 +31,8 @@ import ( v1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/mime" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/utils/cfg" @@ -40,7 +40,7 @@ import ( type cback struct { conf *Options - client *http.Client + client *httpclient.Client } func init() { @@ -55,8 +55,7 @@ func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err erro return nil, err } - httpClient := rhttp.GetHTTPClient() - + httpClient := httpclient.New() // Returns the storage.FS interface return &cback{conf: &o, client: httpClient}, nil } diff --git a/pkg/storage/utils/downloader/downloader.go b/pkg/storage/utils/downloader/downloader.go index badccd1201..4e1efe81c1 100644 --- a/pkg/storage/utils/downloader/downloader.go +++ b/pkg/storage/utils/downloader/downloader.go @@ -30,7 +30,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/httpclient" ) // Downloader is the interface implemented by the objects that are able to @@ -41,14 +41,14 @@ type Downloader interface { type revaDownloader struct { gtw gateway.GatewayAPIClient - httpClient *http.Client + httpClient *httpclient.Client } // NewDownloader creates a Downloader from the reva gateway. -func NewDownloader(gtw gateway.GatewayAPIClient, options ...rhttp.Option) Downloader { +func NewDownloader(gtw gateway.GatewayAPIClient, hc *httpclient.Client) Downloader { return &revaDownloader{ gtw: gtw, - httpClient: rhttp.GetHTTPClient(options...), + httpClient: hc, } } @@ -92,7 +92,7 @@ func (r *revaDownloader) Download(ctx context.Context, path, versionKey string) return nil, err } - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.DownloadEndpoint, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, p.DownloadEndpoint, nil) if err != nil { return nil, err } diff --git a/tests/helpers/helpers.go b/tests/helpers/helpers.go index 7a7ab6f641..b86fcdf806 100644 --- a/tests/helpers/helpers.go +++ b/tests/helpers/helpers.go @@ -35,7 +35,6 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" - "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage" "github.com/studio-b12/gowebdav" ) @@ -127,7 +126,7 @@ func UploadGateway(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, ref token, endpoint = p.Token, p.UploadEndpoint } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, endpoint, bytes.NewReader(content)) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint, bytes.NewReader(content)) if err != nil { return errors.Wrap(err, "error creating new request") } @@ -165,7 +164,7 @@ func Download(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, ref *prov token, endpoint = p.Token, p.DownloadEndpoint } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, endpoint, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, err } @@ -236,7 +235,7 @@ func CreateFile(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, path st token, endpoint = p.Token, p.UploadEndpoint } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, endpoint, bytes.NewReader(content)) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint, bytes.NewReader(content)) if err != nil { return err } From a46da116e1816fea711941e3bd34c010f9a22587 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 27 Sep 2023 14:28:40 +0200 Subject: [PATCH 11/25] tidy go.mod --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 7b57e9b818..d483b8b2d8 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,6 @@ require ( github.com/gocraft/dbr/v2 v2.7.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.1.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect From b6807a019d0e720952e9f5cc15c1fbae6e5290f7 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 27 Sep 2023 14:30:02 +0200 Subject: [PATCH 12/25] remove drone tests --- .drone.env | 4 - .drone.star | 242 ---------------------------------------------------- 2 files changed, 246 deletions(-) delete mode 100644 .drone.env delete mode 100644 .drone.star diff --git a/.drone.env b/.drone.env deleted file mode 100644 index f9010ed6f2..0000000000 --- a/.drone.env +++ /dev/null @@ -1,4 +0,0 @@ -# The test runner source for API tests -APITESTS_COMMITID=5efc3c51a17e5c71a366436afeb52528ed49c7da -APITESTS_BRANCH=master -APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/.drone.star b/.drone.star deleted file mode 100644 index d90a8f9836..0000000000 --- a/.drone.star +++ /dev/null @@ -1,242 +0,0 @@ -OC_CI_GOLANG = "owncloudci/golang:1.19" -OC_CI_ALPINE = "owncloudci/alpine:latest" -OSIXIA_OPEN_LDAP = "osixia/openldap:1.3.0" -OC_CI_PHP = "cs3org/behat:latest" -OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" - -def makeStep(): - return { - "name": "build", - "image": OC_CI_GOLANG, - "commands": [ - "make revad", - ], - } - -def cloneApiTestReposStep(): - return { - "name": "clone-api-test-repos", - "image": OC_CI_ALPINE, - "commands": [ - "source /drone/src/.drone.env", - "git clone -b master --depth=1 https://github.com/owncloud/testing.git /drone/src/tmp/testing", - "git clone -b $APITESTS_BRANCH --single-branch --no-tags $APITESTS_REPO_GIT_URL /drone/src/tmp/testrunner", - "cd /drone/src/tmp/testrunner", - "git checkout $APITESTS_COMMITID", - ], - } - -# Shared service definitions -def ldapService(): - return { - "name": "ldap", - "image": OSIXIA_OPEN_LDAP, - "pull": "always", - "environment": { - "LDAP_DOMAIN": "owncloud.com", - "LDAP_ORGANISATION": "ownCloud", - "LDAP_ADMIN_PASSWORD": "admin", - "LDAP_TLS_VERIFY_CLIENT": "never", - "HOSTNAME": "ldap", - }, - } - -def cephService(): - return { - "name": "ceph", - "image": "ceph/daemon", - "pull": "always", - "environment": { - "CEPH_DAEMON": "demo", - "NETWORK_AUTO_DETECT": "4", - "MON_IP": "0.0.0.0", - "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", - "RGW_CIVETWEB_PORT": "4000 ", - "RGW_NAME": "ceph", - "CEPH_DEMO_UID": "test-user", - "CEPH_DEMO_ACCESS_KEY": "test", - "CEPH_DEMO_SECRET_KEY": "test", - "CEPH_DEMO_BUCKET": "test", - }, - } - -# Pipeline definitions -def main(ctx): - return [ - checkStarlark(), - ocisIntegrationTest(), - ] + s3ngIntegrationTests() - -def ocisIntegrationTest(): - return { - "kind": "pipeline", - "type": "docker", - "name": "ocis-integration-tests-2", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep(), - { - "name": "revad-services", - "image": OC_CI_GOLANG, - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c shares.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", - "/drone/src/cmd/revad/revad -c ldap-users.toml", - ], - }, - cloneApiTestReposStep(), - { - "name": "APIAcceptanceTestsOcisStorage", - "image": OC_CI_PHP, - "commands": [ - "cd /drone/src/tmp/testrunner", - "make test-acceptance-from-core-api", - ], - "environment": { - "TEST_SERVER_URL": "http://revad-services:20080", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "OCIS", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_WITH_LDAP": "true", - "REVA_LDAP_HOSTNAME": "ldap", - "TEST_REVA": "true", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", - "DIVIDE_INTO_NUM_PARTS": 6, - "RUN_PART": 2, - "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", - }, - }, - ], - "services": [ - ldapService(), - ], - } - -def s3ngIntegrationTests(): - parallelRuns = 12 - pipelines = [] - for runPart in range(1, parallelRuns + 1): - if runPart in [9]: - continue - - pipelines.append( - { - "kind": "pipeline", - "type": "docker", - "name": "s3ng-integration-tests-%s" % runPart, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep(), - { - "name": "revad-services", - "image": OC_CI_GOLANG, - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c shares.toml &", - "/drone/src/cmd/revad/revad -c storage-home-s3ng.toml &", - "/drone/src/cmd/revad/revad -c storage-users-s3ng.toml &", - "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", - "/drone/src/cmd/revad/revad -c ldap-users.toml", - ], - }, - cloneApiTestReposStep(), - { - "name": "APIAcceptanceTestsS3ngStorage", - "image": OC_CI_PHP, - "commands": [ - "cd /drone/src/tmp/testrunner", - "make test-acceptance-from-core-api", - ], - "environment": { - "TEST_SERVER_URL": "http://revad-services:20080", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "S3NG", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_WITH_LDAP": "true", - "REVA_LDAP_HOSTNAME": "ldap", - "TEST_REVA": "true", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", - "DIVIDE_INTO_NUM_PARTS": parallelRuns, - "RUN_PART": runPart, - "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", - }, - }, - ], - "services": [ - ldapService(), - cephService(), - ], - }, - ) - - return pipelines - -def checkStarlark(): - return { - "kind": "pipeline", - "type": "docker", - "name": "check-starlark", - "steps": [ - { - "name": "format-check-starlark", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=check .drone.star", - ], - }, - { - "name": "show-diff", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=fix .drone.star", - "git diff", - ], - "when": { - "status": [ - "failure", - ], - }, - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - } From 94e77487f2b301e0ccad91856f2f467bac8ef320 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 27 Sep 2023 15:12:35 +0200 Subject: [PATCH 13/25] add changelog --- changelog/unreleased/observability.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/observability.md diff --git a/changelog/unreleased/observability.md b/changelog/unreleased/observability.md new file mode 100644 index 0000000000..9bb1640c61 --- /dev/null +++ b/changelog/unreleased/observability.md @@ -0,0 +1,6 @@ +Enhancement: Add better observability with metrics and traces + +Adds prometheus collectors that can be registered dynamically and also +refactors the http and grpc clients and servers to propage trace info. + +https://github.com/cs3org/reva/pull/4166 From 210803d970f60858b8f0cabdebd62a9cc977bf5b Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 27 Sep 2023 15:16:50 +0200 Subject: [PATCH 14/25] fix test --- tests/integration/grpc/ocm_share_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/grpc/ocm_share_test.go b/tests/integration/grpc/ocm_share_test.go index 614d26aef4..ffe5f00f37 100644 --- a/tests/integration/grpc/ocm_share_test.go +++ b/tests/integration/grpc/ocm_share_test.go @@ -37,7 +37,6 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/tests/helpers" . "github.com/onsi/ginkgo" @@ -700,7 +699,7 @@ func download(ctx context.Context, gw gatewaypb.GatewayAPIClient, ref *provider. token, endpoint = p.Token, p.DownloadEndpoint } } - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, endpoint, nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, err } From 6fb11d1490a010fcb0bf1c11699c3194be2ea72c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 11:10:58 +0200 Subject: [PATCH 15/25] go 1.21 --- go.mod | 2 +- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8b06b8fee2..b5df636545 100644 --- a/go.mod +++ b/go.mod @@ -159,7 +159,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.20 +go 1.21 replace ( github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a diff --git a/go.sum b/go.sum index 7f83bd3ab2..596abae755 100644 --- a/go.sum +++ b/go.sum @@ -1091,6 +1091,7 @@ github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KA github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -1106,6 +1107,7 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= @@ -1414,6 +1416,7 @@ github.com/lestrrat-go/strftime v1.0.4/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR7 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.25.3/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= @@ -1557,6 +1560,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -1675,6 +1679,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -1709,6 +1714,7 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY= +github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= From c0c86fc0a04c3f1a666ea2a2f2a21a6a63db2691 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 19:07:25 +0200 Subject: [PATCH 16/25] pass tests --- internal/grpc/interceptors/appctx/appctx.go | 4 +-- internal/grpc/interceptors/trace/trace.go | 9 ++++-- internal/grpc/services/pingpong/pingpong.go | 34 ++++++++++++++++++--- internal/http/interceptors/appctx/appctx.go | 2 +- internal/http/interceptors/trace/trace.go | 2 +- pkg/appctx/logger.go | 5 +-- pkg/httpclient/httpclient.go | 1 - pkg/ocm/client/client.go | 8 ++--- pkg/ocm/storage/outcoming/ocm.go | 5 +-- pkg/trace/trace.go | 6 ++-- tests/helpers/helpers.go | 7 +++-- tests/integration/grpc/ocm_share_test.go | 3 +- 12 files changed, 56 insertions(+), 30 deletions(-) diff --git a/internal/grpc/interceptors/appctx/appctx.go b/internal/grpc/interceptors/appctx/appctx.go index 39a8a410d8..9aa0201aa3 100644 --- a/internal/grpc/interceptors/appctx/appctx.go +++ b/internal/grpc/interceptors/appctx/appctx.go @@ -31,7 +31,7 @@ import ( func NewUnary(log zerolog.Logger) grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { traceID := trace.Get(ctx) - log = log.With().Str(trace.Key, traceID).Logger() + log := log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) res, err := handler(ctx, req) return res, err @@ -45,7 +45,7 @@ func NewStream(log zerolog.Logger) grpc.StreamServerInterceptor { interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() traceID := trace.Get(ctx) - log = log.With().Str(trace.Key, traceID).Logger() + log := log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) wrapped := newWrappedServerStream(ctx, ss) diff --git a/internal/grpc/interceptors/trace/trace.go b/internal/grpc/interceptors/trace/trace.go index 2f94c6a3bb..345497a0bd 100644 --- a/internal/grpc/interceptors/trace/trace.go +++ b/internal/grpc/interceptors/trace/trace.go @@ -27,10 +27,15 @@ import ( ) func getContext(ctx context.Context) context.Context { - var traceID string + traceID := trace.Get(ctx) + if traceID != "" { + return ctx + + } + md, ok := metadata.FromIncomingContext(ctx) if ok && md != nil { - if val, ok := md[trace.Key]; ok { + if val, ok := md["revad-grpc-trace-id"]; ok { if len(val) > 0 && val[0] != "" { traceID = val[0] } diff --git a/internal/grpc/services/pingpong/pingpong.go b/internal/grpc/services/pingpong/pingpong.go index 564a3daff4..f8f94ea7f7 100644 --- a/internal/grpc/services/pingpong/pingpong.go +++ b/internal/grpc/services/pingpong/pingpong.go @@ -82,7 +82,7 @@ func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingRe } client := httpclient.New(httpclient.RoundTripper(tr)) - req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) + req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/ping", nil) if err != nil { log.Error().Err(err).Msg("error creating http request") return nil, err @@ -104,7 +104,33 @@ func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingRe return &proto.PingResponse{Info: string(data)}, nil } -func (s *service) Pong(_ context.Context, _ *proto.PongRequest) (*proto.PongResponse, error) { - res := &proto.PongResponse{Info: "pong"} - return res, nil +func (s *service) Pong(ctx context.Context, _ *proto.PongRequest) (*proto.PongResponse, error) { + log := appctx.GetLogger(ctx) + + // we call the http Pong method + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + client := httpclient.New(httpclient.RoundTripper(tr)) + req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) + if err != nil { + log.Error().Err(err).Msg("error creating http request") + return nil, err + } + + res, err := client.Do(req) + if err != nil { + log.Error().Err(err).Msg("eror doing http pong") + return nil, err + } + + defer res.Body.Close() + data, err := ioutil.ReadAll(res.Body) + if err != nil { + log.Error().Err(err).Msg("error reading response body") + return nil, err + } + + return &proto.PongResponse{Info: string(data)}, nil } diff --git a/internal/http/interceptors/appctx/appctx.go b/internal/http/interceptors/appctx/appctx.go index faf994135b..6dd2511d3c 100644 --- a/internal/http/interceptors/appctx/appctx.go +++ b/internal/http/interceptors/appctx/appctx.go @@ -42,7 +42,7 @@ func handler(log zerolog.Logger, h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() traceID := trace.Get(ctx) - log = log.With().Str(trace.Key, traceID).Logger() + log := log.With().Str("traceid", traceID).Logger() ctx = appctx.WithLogger(ctx, &log) r = r.WithContext(ctx) h.ServeHTTP(w, r) diff --git a/internal/http/interceptors/trace/trace.go b/internal/http/interceptors/trace/trace.go index 25a83a4e9a..f4bec73216 100644 --- a/internal/http/interceptors/trace/trace.go +++ b/internal/http/interceptors/trace/trace.go @@ -54,7 +54,7 @@ func handler(h http.Handler) http.Handler { // in case the http service will call a grpc service, // we set the outgoing context so the trace information is // passed through the two protocols. - ctx = metadata.AppendToOutgoingContext(ctx, trace.Key, traceID) + ctx = metadata.AppendToOutgoingContext(ctx, "revad-grpc-trace-id", traceID) r = r.WithContext(ctx) h.ServeHTTP(w, r) }) diff --git a/pkg/appctx/logger.go b/pkg/appctx/logger.go index a708956cd3..8624725ea5 100644 --- a/pkg/appctx/logger.go +++ b/pkg/appctx/logger.go @@ -21,15 +21,12 @@ package appctx import ( "context" - "github.com/cs3org/reva/pkg/trace" "github.com/rs/zerolog" ) // WithLogger returns a context with an associated logger. func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context { - traceID := trace.Get(ctx) - sublog := l.With().Str(trace.Key, traceID).Logger() - return sublog.WithContext(ctx) + return l.WithContext(ctx) } // GetLogger returns the logger associated with the given context diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index 130e8bd2be..d4ee86d6b6 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -107,7 +107,6 @@ type injectTransport struct { } func (t injectTransport) RoundTrip(r *http.Request) (*http.Response, error) { - // assume the request context has not been populated with tracing information. ctx := r.Context() traceID := trace.Get(ctx) diff --git a/pkg/ocm/client/client.go b/pkg/ocm/client/client.go index 5c735678e7..07c4e1265e 100644 --- a/pkg/ocm/client/client.go +++ b/pkg/ocm/client/client.go @@ -31,7 +31,6 @@ import ( "github.com/cs3org/reva/internal/http/services/ocmd" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/httpclient" "github.com/pkg/errors" ) @@ -57,7 +56,7 @@ var ErrInvalidParameters = errors.New("invalid parameters") // OCMClient is the client for an OCM provider. type OCMClient struct { - client *httpclient.Client + client *http.Client } // Config is the configuration to be used for the OCMClient. @@ -72,10 +71,7 @@ func New(c *Config) *OCMClient { TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Insecure}, } return &OCMClient{ - client: httpclient.New( - httpclient.Timeout(c.Timeout), - httpclient.RoundTripper(tr), - ), + client: &http.Client{Transport: tr}, } } diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 6397f8f398..21d443cd8e 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -36,6 +36,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" @@ -416,7 +417,7 @@ func (d *driver) Upload(ctx context.Context, ref *provider.Reference, content io httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) + httpRes, err := httpclient.New().Do(httpReq) if err != nil { return errors.Wrap(err, "error doing put request") } @@ -470,7 +471,7 @@ func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.Read } httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) //nolint:golint,bodyclose + httpRes, err := httpclient.New().Do(httpReq) //nolint:golint,bodyclose if err != nil { return err } diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index 64c9a4ef09..233a9c35c5 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -23,10 +23,10 @@ import ( "github.com/gofrs/uuid" ) -const Key string = "traceid" +type key struct{} func Get(ctx context.Context) (t string) { - t, _ = ctx.Value(Key).(string) + t, _ = ctx.Value(key{}).(string) return } @@ -36,5 +36,5 @@ func Generate() string { // ContextSetTrace stores the trace in the context. func Set(ctx context.Context, trace string) context.Context { - return context.WithValue(ctx, Key, trace) + return context.WithValue(ctx, key{}, trace) } diff --git a/tests/helpers/helpers.go b/tests/helpers/helpers.go index b86fcdf806..bf0d69804d 100644 --- a/tests/helpers/helpers.go +++ b/tests/helpers/helpers.go @@ -35,6 +35,7 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/storage" "github.com/studio-b12/gowebdav" ) @@ -133,7 +134,7 @@ func UploadGateway(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, ref httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) + httpRes, err := httpclient.New().Do(httpReq) if err != nil { return errors.Wrap(err, "error doing put request") } @@ -170,7 +171,7 @@ func Download(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, ref *prov } httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) + httpRes, err := httpclient.New().Do(httpReq) if err != nil { return nil, err } @@ -242,7 +243,7 @@ func CreateFile(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, path st httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) + httpRes, err := httpclient.New().Do(httpReq) if err != nil { return err } diff --git a/tests/integration/grpc/ocm_share_test.go b/tests/integration/grpc/ocm_share_test.go index ffe5f00f37..f34ddf2ff9 100644 --- a/tests/integration/grpc/ocm_share_test.go +++ b/tests/integration/grpc/ocm_share_test.go @@ -35,6 +35,7 @@ import ( "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" + "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" @@ -706,7 +707,7 @@ func download(ctx context.Context, gw gatewaypb.GatewayAPIClient, ref *provider. httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := http.DefaultClient.Do(httpReq) + httpRes, err := httpclient.New().Do(httpReq) if err != nil { return nil, err } From 861f02887d8eca8add1bc5bb97ce9e412d08be47 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 19:14:18 +0200 Subject: [PATCH 17/25] fix sql tests --- pkg/share/manager/sql/conversions.go | 262 +++++++++++ pkg/share/manager/sql/mocks/UserConverter.go | 78 ++++ pkg/share/manager/sql/sql.go | 431 +++++++++---------- pkg/share/manager/sql/sql_suite_test.go | 31 ++ pkg/share/manager/sql/sql_test.go | 6 +- pkg/share/manager/sql/test.db | Bin 0 -> 655360 bytes 6 files changed, 572 insertions(+), 236 deletions(-) create mode 100644 pkg/share/manager/sql/conversions.go create mode 100644 pkg/share/manager/sql/mocks/UserConverter.go create mode 100644 pkg/share/manager/sql/sql_suite_test.go create mode 100644 pkg/share/manager/sql/test.db diff --git a/pkg/share/manager/sql/conversions.go b/pkg/share/manager/sql/conversions.go new file mode 100644 index 0000000000..a09cae34e5 --- /dev/null +++ b/pkg/share/manager/sql/conversions.go @@ -0,0 +1,262 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sql + +import ( + "context" + + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + conversions "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" + "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" +) + +//go:generate mockery -name UserConverter + +// DBShare stores information about user and public shares. +type DBShare struct { + ID string + UIDOwner string + UIDInitiator string + ItemStorage string + ItemSource string + ShareWith string + Token string + Expiration string + Permissions int + ShareType int + ShareName string + STime int + FileTarget string + RejectedBy string + State int +} + +// UserConverter describes an interface for converting user ids to names and back. +type UserConverter interface { + UserNameToUserID(ctx context.Context, username string) (*userpb.UserId, error) + UserIDToUserName(ctx context.Context, userid *userpb.UserId) (string, error) +} + +// GatewayUserConverter converts usernames and ids using the gateway. +type GatewayUserConverter struct { + gwAddr string +} + +// NewGatewayUserConverter returns a instance of GatewayUserConverter. +func NewGatewayUserConverter(gwAddr string) *GatewayUserConverter { + return &GatewayUserConverter{ + gwAddr: gwAddr, + } +} + +// UserIDToUserName converts a user ID to an username. +func (c *GatewayUserConverter) UserIDToUserName(ctx context.Context, userid *userpb.UserId) (string, error) { + gwConn, err := pool.GetGatewayServiceClient(pool.Endpoint(c.gwAddr)) + if err != nil { + return "", err + } + getUserResponse, err := gwConn.GetUser(ctx, &userpb.GetUserRequest{ + UserId: userid, + SkipFetchingUserGroups: true, + }) + if err != nil { + return "", err + } + if getUserResponse.Status.Code != rpc.Code_CODE_OK { + return "", status.NewErrorFromCode(getUserResponse.Status.Code, "gateway") + } + return getUserResponse.User.Username, nil +} + +// UserNameToUserID converts a username to an user ID. +func (c *GatewayUserConverter) UserNameToUserID(ctx context.Context, username string) (*userpb.UserId, error) { + gwConn, err := pool.GetGatewayServiceClient(pool.Endpoint(c.gwAddr)) + if err != nil { + return nil, err + } + getUserResponse, err := gwConn.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{ + Claim: "username", + Value: username, + SkipFetchingUserGroups: true, + }) + if err != nil { + return nil, err + } + if getUserResponse.Status.Code != rpc.Code_CODE_OK { + return nil, status.NewErrorFromCode(getUserResponse.Status.Code, "gateway") + } + return getUserResponse.User.Id, nil +} + +func (m *mgr) formatGrantee(ctx context.Context, g *provider.Grantee) (int, string, error) { + var granteeType int + var formattedID string + switch g.Type { + case provider.GranteeType_GRANTEE_TYPE_USER: + granteeType = 0 + var err error + formattedID, err = m.userConverter.UserIDToUserName(ctx, g.GetUserId()) + if err != nil { + return 0, "", err + } + case provider.GranteeType_GRANTEE_TYPE_GROUP: + granteeType = 1 + formattedID = formatGroupID(g.GetGroupId()) + default: + granteeType = -1 + } + return granteeType, formattedID, nil +} + +func (m *mgr) extractGrantee(ctx context.Context, t int, g string) (*provider.Grantee, error) { + var grantee provider.Grantee + switch t { + case 0: + userid, err := m.userConverter.UserNameToUserID(ctx, g) + if err != nil { + return nil, err + } + grantee.Type = provider.GranteeType_GRANTEE_TYPE_USER + grantee.Id = &provider.Grantee_UserId{UserId: userid} + case 1: + grantee.Type = provider.GranteeType_GRANTEE_TYPE_GROUP + grantee.Id = &provider.Grantee_GroupId{GroupId: extractGroupID(g)} + default: + grantee.Type = provider.GranteeType_GRANTEE_TYPE_INVALID + } + return &grantee, nil +} + +func resourceTypeToItem(r provider.ResourceType) string { + switch r { + case provider.ResourceType_RESOURCE_TYPE_FILE: + return "file" + case provider.ResourceType_RESOURCE_TYPE_CONTAINER: + return "folder" + case provider.ResourceType_RESOURCE_TYPE_REFERENCE: + return "reference" + case provider.ResourceType_RESOURCE_TYPE_SYMLINK: + return "symlink" + default: + return "" + } +} + +func sharePermToInt(p *provider.ResourcePermissions) int { + return int(conversions.RoleFromResourcePermissions(p).OCSPermissions()) +} + +func intTosharePerm(p int) (*provider.ResourcePermissions, error) { + perms, err := conversions.NewPermissions(p) + if err != nil { + return nil, err + } + + return conversions.RoleFromOCSPermissions(perms).CS3ResourcePermissions(), nil +} + +func intToShareState(g int) collaboration.ShareState { + switch g { + case 0: + return collaboration.ShareState_SHARE_STATE_ACCEPTED + case 1: + return collaboration.ShareState_SHARE_STATE_PENDING + case 2: + return collaboration.ShareState_SHARE_STATE_REJECTED + default: + return collaboration.ShareState_SHARE_STATE_INVALID + } +} + +func formatUserID(u *userpb.UserId) string { + return u.OpaqueId +} + +func formatGroupID(u *grouppb.GroupId) string { + return u.OpaqueId +} + +func extractGroupID(u string) *grouppb.GroupId { + return &grouppb.GroupId{OpaqueId: u} +} + +func (m *mgr) convertToCS3Share(ctx context.Context, s DBShare, storageMountID string) (*collaboration.Share, error) { + ts := &typespb.Timestamp{ + Seconds: uint64(s.STime), + } + permissions, err := intTosharePerm(s.Permissions) + if err != nil { + return nil, err + } + grantee, err := m.extractGrantee(ctx, s.ShareType, s.ShareWith) + if err != nil { + return nil, err + } + owner, err := m.userConverter.UserNameToUserID(ctx, s.UIDOwner) + if err != nil { + return nil, err + } + var creator *userpb.UserId + if s.UIDOwner == s.UIDInitiator { + creator = owner + } else { + creator, err = m.userConverter.UserNameToUserID(ctx, s.UIDOwner) + if err != nil { + return nil, err + } + } + return &collaboration.Share{ + Id: &collaboration.ShareId{ + OpaqueId: s.ID, + }, + ResourceId: &provider.ResourceId{ + StorageId: storageMountID + "!" + s.ItemStorage, + OpaqueId: s.ItemSource, + }, + Permissions: &collaboration.SharePermissions{Permissions: permissions}, + Grantee: grantee, + Owner: owner, + Creator: creator, + Ctime: ts, + Mtime: ts, + }, nil +} + +func (m *mgr) convertToCS3ReceivedShare(ctx context.Context, s DBShare, storageMountID string) (*collaboration.ReceivedShare, error) { + share, err := m.convertToCS3Share(ctx, s, storageMountID) + if err != nil { + return nil, err + } + var state collaboration.ShareState + if s.RejectedBy != "" { + state = collaboration.ShareState_SHARE_STATE_REJECTED + } else { + state = intToShareState(s.State) + } + return &collaboration.ReceivedShare{ + Share: share, + State: state, + }, nil +} diff --git a/pkg/share/manager/sql/mocks/UserConverter.go b/pkg/share/manager/sql/mocks/UserConverter.go new file mode 100644 index 0000000000..48da015a80 --- /dev/null +++ b/pkg/share/manager/sql/mocks/UserConverter.go @@ -0,0 +1,78 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" +) + +// UserConverter is an autogenerated mock type for the UserConverter type +type UserConverter struct { + mock.Mock +} + +// UserIDToUserName provides a mock function with given fields: ctx, userid +func (_m *UserConverter) UserIDToUserName(ctx context.Context, userid *userv1beta1.UserId) (string, error) { + ret := _m.Called(ctx, userid) + + var r0 string + if rf, ok := ret.Get(0).(func(context.Context, *userv1beta1.UserId) string); ok { + r0 = rf(ctx, userid) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *userv1beta1.UserId) error); ok { + r1 = rf(ctx, userid) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// UserNameToUserID provides a mock function with given fields: ctx, username +func (_m *UserConverter) UserNameToUserID(ctx context.Context, username string) (*userv1beta1.UserId, error) { + ret := _m.Called(ctx, username) + + var r0 *userv1beta1.UserId + if rf, ok := ret.Get(0).(func(context.Context, string) *userv1beta1.UserId); ok { + r0 = rf(ctx, username) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*userv1beta1.UserId) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, username) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/share/manager/sql/sql.go b/pkg/share/manager/sql/sql.go index d88d4a9232..2c156fd713 100644 --- a/pkg/share/manager/sql/sql.go +++ b/pkg/share/manager/sql/sql.go @@ -27,15 +27,11 @@ import ( "strings" "time" - gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - conversions "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" "github.com/cs3org/reva/pkg/sharedconf" @@ -51,37 +47,35 @@ import ( const ( shareTypeUser = 0 shareTypeGroup = 1 - - projectInstancesPrefix = "newproject" - projectSpaceGroupsPrefix = "cernbox-project-" - projectSpaceAdminGroupsSuffix = "-admins" ) func init() { - registry.Register("sql", New) + registry.Register("oc10-sql", NewMysql) } type config struct { - DBUsername string `mapstructure:"db_username"` - DBPassword string `mapstructure:"db_password"` - DBHost string `mapstructure:"db_host"` - DBPort int `mapstructure:"db_port"` - DBName string `mapstructure:"db_name"` - GatewaySvc string `mapstructure:"gatewaysvc"` + GatewayAddr string `mapstructure:"gatewaysvc"` + StorageMountID string `mapstructure:"storage_mount_id"` + DBUsername string `mapstructure:"db_username"` + DBPassword string `mapstructure:"db_password"` + DBHost string `mapstructure:"db_host"` + DBPort int `mapstructure:"db_port"` + DBName string `mapstructure:"db_name"` } -type mgr struct { - c *config - db *sql.DB - client gatewayv1beta1.GatewayAPIClient +func (c *config) ApplyDefaults() { + c.GatewayAddr = sharedconf.GetGatewaySVC(c.GatewayAddr) } -func (c *config) ApplyDefaults() { - c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) +type mgr struct { + driver string + db *sql.DB + storageMountID string + userConverter UserConverter } -// New returns a new share manager. -func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { +// NewMysql returns a new share manager connection to a mysql database. +func NewMysql(ctx context.Context, m map[string]interface{}) (share.Manager, error) { var c config if err := cfg.Decode(m, &c); err != nil { return nil, err @@ -92,15 +86,18 @@ func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { return nil, err } - gw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) - if err != nil { - return nil, err - } + userConverter := NewGatewayUserConverter(c.GatewayAddr) + + return New("mysql", db, c.StorageMountID, userConverter) +} +// New returns a new Cache instance connecting to the given sql.DB. +func New(driver string, db *sql.DB, storageMountID string, userConverter UserConverter) (share.Manager, error) { return &mgr{ - c: &c, - db: db, - client: gw, + driver: driver, + db: db, + storageMountID: storageMountID, + userConverter: userConverter, }, nil } @@ -132,11 +129,17 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collabora Seconds: uint64(now), } - shareType, shareWith := conversions.FormatGrantee(g.Grantee) - itemType := conversions.ResourceTypeToItem(md.Type) + owner, err := m.userConverter.UserIDToUserName(ctx, md.Owner) + if err != nil { + return nil, err + } + shareType, shareWith, err := m.formatGrantee(ctx, g.Grantee) + if err != nil { + return nil, err + } + itemType := resourceTypeToItem(md.Type) targetPath := path.Join("/", path.Base(md.Path)) - permissions := conversions.SharePermToInt(g.Permissions.Permissions) - prefix := md.Id.StorageId + permissions := sharePermToInt(g.Permissions.Permissions) itemSource := md.Id.OpaqueId fileSource, err := strconv.ParseUint(itemSource, 10, 64) if err != nil { @@ -145,8 +148,8 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collabora fileSource = 0 } - stmtString := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,share_with=?,file_target=?" - stmtValues := []interface{}{shareType, conversions.FormatUserID(md.Owner), conversions.FormatUserID(user.Id), itemType, prefix, itemSource, fileSource, permissions, now, shareWith, targetPath} + stmtString := "INSERT INTO oc_share (share_type,uid_owner,uid_initiator,item_type,item_source,file_source,permissions,stime,share_with,file_target) VALUES (?,?,?,?,?,?,?,?,?,?)" + stmtValues := []interface{}{shareType, owner, user.Username, itemType, itemSource, fileSource, permissions, now, shareWith, targetPath} stmt, err := m.db.Prepare(stmtString) if err != nil { @@ -175,35 +178,6 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collabora }, nil } -func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) { - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - s := conversions.DBShare{ID: id.OpaqueId} - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND id=? AND (uid_owner=? or uid_initiator=?)" - if err := m.db.QueryRow(query, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(id.OpaqueId) - } - return nil, err - } - return conversions.ConvertToCS3Share(s), nil -} - -func (m *mgr) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) { - owner := conversions.FormatUserID(key.Owner) - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - - s := conversions.DBShare{} - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, id, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - if err := m.db.QueryRow(query, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(key.String()) - } - return nil, err - } - return conversions.ConvertToCS3Share(s), nil -} - func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { var s *collaboration.Share var err error @@ -224,19 +198,22 @@ func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) ( } func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) + uid := appctx.ContextMustGetUser(ctx).Username var query string params := []interface{}{} switch { case ref.GetId() != nil: - query = "delete from oc_share where id=? AND (uid_owner=? or uid_initiator=?)" + query = "DELETE FROM oc_share where id=? AND (uid_owner=? or uid_initiator=?)" params = append(params, ref.GetId().OpaqueId, uid, uid) case ref.GetKey() != nil: key := ref.GetKey() - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - owner := conversions.FormatUserID(key.Owner) - query = "delete from oc_share where uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid) + shareType, shareWith, err := m.formatGrantee(ctx, key.Grantee) + if err != nil { + return err + } + owner := formatUserID(key.Owner) + query = "DELETE FROM oc_share WHERE uid_owner=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" + params = append(params, owner, key.ResourceId.StorageId, shareType, shareWith, uid, uid) default: return errtypes.NotFound(ref.String()) } @@ -261,8 +238,8 @@ func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) er } func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { - permissions := conversions.SharePermToInt(p.Permissions) - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) + permissions := sharePermToInt(p.Permissions) + uid := appctx.ContextMustGetUser(ctx).Username var query string params := []interface{}{} @@ -272,10 +249,13 @@ func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference params = append(params, permissions, time.Now().Unix(), ref.GetId().OpaqueId, uid, uid) case ref.GetKey() != nil: key := ref.GetKey() - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - owner := conversions.FormatUserID(key.Owner) - query = "update oc_share set permissions=?,stime=? where (uid_owner=? or uid_initiator=?) AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, permissions, time.Now().Unix(), owner, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid) + shareType, shareWith, err := m.formatGrantee(ctx, key.Grantee) + if err != nil { + return nil, err + } + owner := formatUserID(key.Owner) + query = "update oc_share set permissions=?,stime=? where (uid_owner=? or uid_initiator=?) AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" + params = append(params, permissions, time.Now().Unix(), owner, owner, key.ResourceId.StorageId, shareType, shareWith, uid, uid) default: return nil, errtypes.NotFound(ref.String()) } @@ -292,31 +272,29 @@ func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference } func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - id, stime, permissions, share_type - FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (share_type=? OR share_type=?)` - params := []interface{}{shareTypeUser, shareTypeGroup} + uid := appctx.ContextMustGetUser(ctx).Username + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, id, stime, permissions, share_type FROM oc_share WHERE (uid_owner=? or uid_initiator=?)" + params := []interface{}{uid, uid} - groupedFilters := share.GroupFiltersByType(filters) - if len(groupedFilters) > 0 { - filterQuery, filterParams, err := translateFilters(groupedFilters) + var ( + filterQuery string + filterParams []interface{} + err error + ) + if len(filters) == 0 { + filterQuery += "(share_type=? OR share_type=?)" + params = append(params, shareTypeUser) + params = append(params, shareTypeGroup) + } else { + filterQuery, filterParams, err = translateFilters(filters) if err != nil { return nil, err } params = append(params, filterParams...) - if filterQuery != "" { - query = fmt.Sprintf("%s AND (%s)", query, filterQuery) - } } - uidOwnersQuery, uidOwnersParams, err := m.uidOwnerFilters(ctx, groupedFilters) - if err != nil { - return nil, err - } - params = append(params, uidOwnersParams...) - if uidOwnersQuery != "" { - query = fmt.Sprintf("%s AND (%s)", query, uidOwnersQuery) + if filterQuery != "" { + query = fmt.Sprintf("%s AND (%s)", query, filterQuery) } rows, err := m.db.Query(query, params...) @@ -325,13 +303,17 @@ func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ( } defer rows.Close() - var s conversions.DBShare + var s DBShare shares := []*collaboration.Share{} for rows.Next() { - if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { + if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { continue } - shares = append(shares, conversions.ConvertToCS3Share(s)) + share, err := m.convertToCS3Share(ctx, s, m.storageMountID) + if err != nil { + return nil, err + } + shares = append(shares, share) } if err = rows.Err(); err != nil { return nil, err @@ -343,26 +325,27 @@ func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ( // we list the shares that are targeted to the user in context or to the user groups. func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) + uid := user.Username - params := []interface{}{uid, uid, uid, uid} + params := []interface{}{uid, uid, uid} for _, v := range user.Groups { params = append(params, v) } - query := `SELECT coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner != ? AND uid_initiator != ?)` + homeConcat := "" + if m.driver == "mysql" { // mysql upsert + homeConcat = "storages.id = CONCAT('home::', ts.uid_owner)" + } else { // sqlite3 upsert + homeConcat = "storages.id = 'home::' || ts.uid_owner" + } + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, ts.id, stime, permissions, share_type, accepted, storages.numeric_id FROM oc_share ts LEFT JOIN oc_storages storages ON " + homeConcat + " WHERE (uid_owner != ? AND uid_initiator != ?) " if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" + query += "AND (share_with=? OR share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + "))" } else { - query += " AND (share_with=? AND share_type = 0)" + query += "AND (share_with=?)" } - groupedFilters := share.GroupFiltersByType(filters) - filterQuery, filterParams, err := translateFilters(groupedFilters) + filterQuery, filterParams, err := translateFilters(filters) if err != nil { return nil, err } @@ -378,13 +361,17 @@ func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.F } defer rows.Close() - var s conversions.DBShare + var s DBShare shares := []*collaboration.ReceivedShare{} for rows.Next() { - if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { + if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State, &s.ItemStorage); err != nil { continue } - shares = append(shares, conversions.ConvertToCS3ReceivedShare(s)) + share, err := m.convertToCS3ReceivedShare(ctx, s, m.storageMountID) + if err != nil { + return nil, err + } + shares = append(shares, share) } if err = rows.Err(); err != nil { return nil, err @@ -393,67 +380,6 @@ func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.F return shares, nil } -func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - params := []interface{}{uid, id.OpaqueId, uid} - for _, v := range user.Groups { - params = append(params, v) - } - - s := conversions.DBShare{ID: id.OpaqueId} - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND ts.id=?` - if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" - } else { - query += " AND (share_with=? AND share_type = 0)" - } - if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(id.OpaqueId) - } - return nil, err - } - return conversions.ConvertToCS3ReceivedShare(s), nil -} - -func (m *mgr) getReceivedByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - params := []interface{}{uid, conversions.FormatUserID(key.Owner), key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, shareWith} - for _, v := range user.Groups { - params = append(params, v) - } - - s := conversions.DBShare{} - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=?` - if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" - } else { - query += " AND (share_with=? AND share_type = 0)" - } - - if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(key.String()) - } - return nil, err - } - - return conversions.ConvertToCS3ReceivedShare(s), nil -} - func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { var s *collaboration.ReceivedShare var err error @@ -474,8 +400,6 @@ func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareRefe } func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - rs, err := m.GetReceivedShare(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Share.Id}}) if err != nil { return nil, err @@ -485,82 +409,122 @@ func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.Rece switch fieldMask.Paths[i] { case "state": rs.State = share.State + // TODO case "mount_point": default: return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported") } } - state := 0 + var queryAccept string switch rs.GetState() { case collaboration.ShareState_SHARE_STATE_REJECTED: - state = -1 + queryAccept = "update oc_share set accepted=2 where id=?" case collaboration.ShareState_SHARE_STATE_ACCEPTED: - state = 1 + queryAccept = "update oc_share set accepted=0 where id=?" + } + + if queryAccept != "" { + stmt, err := m.db.Prepare(queryAccept) + if err != nil { + return nil, err + } + _, err = stmt.Exec(rs.Share.Id.OpaqueId) + if err != nil { + return nil, err + } } - params := []interface{}{rs.Share.Id.OpaqueId, conversions.FormatUserID(user.Id), state, state} - query := "insert into oc_share_status(id, recipient, state) values(?, ?, ?) ON DUPLICATE KEY UPDATE state = ?" + return rs, nil +} - stmt, err := m.db.Prepare(query) - if err != nil { +func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) { + uid := appctx.ContextMustGetUser(ctx).Username + s := DBShare{ID: id.OpaqueId} + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, stime, permissions, share_type FROM oc_share WHERE id=? AND (uid_owner=? or uid_initiator=?)" + if err := m.db.QueryRow(query, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.STime, &s.Permissions, &s.ShareType); err != nil { + if err == sql.ErrNoRows { + return nil, errtypes.NotFound(id.OpaqueId) + } return nil, err } - _, err = stmt.Exec(params...) + return m.convertToCS3Share(ctx, s, m.storageMountID) +} + +func (m *mgr) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) { + owner, err := m.userConverter.UserIDToUserName(ctx, key.Owner) if err != nil { return nil, err } + uid := appctx.ContextMustGetUser(ctx).Username - return rs, nil + s := DBShare{} + shareType, shareWith, err := m.formatGrantee(ctx, key.Grantee) + if err != nil { + return nil, err + } + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, id, stime, permissions, share_type FROM oc_share WHERE uid_owner=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" + if err = m.db.QueryRow(query, owner, key.ResourceId.StorageId, shareType, shareWith, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { + if err == sql.ErrNoRows { + return nil, errtypes.NotFound(key.String()) + } + return nil, err + } + return m.convertToCS3Share(ctx, s, m.storageMountID) } -func (m *mgr) uidOwnerFilters(ctx context.Context, filters map[collaboration.Filter_Type][]*collaboration.Filter) (string, []interface{}, error) { +func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.ReceivedShare, error) { user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) + uid := user.Username - query := "uid_owner=? or uid_initiator=?" - params := []interface{}{uid, uid} + params := []interface{}{id.OpaqueId, uid} + for _, v := range user.Groups { + params = append(params, v) + } - client, err := pool.GetGatewayServiceClient(pool.Endpoint(m.c.GatewaySvc)) - if err != nil { - return "", nil, err + s := DBShare{ID: id.OpaqueId} + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, stime, permissions, share_type, accepted FROM oc_share ts WHERE ts.id=? " + if len(user.Groups) > 0 { + query += "AND (share_with=? OR share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + "))" + } else { + query += "AND (share_with=?)" + } + if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { + if err == sql.ErrNoRows { + return nil, errtypes.NotFound(id.OpaqueId) + } + return nil, err } + return m.convertToCS3ReceivedShare(ctx, s, m.storageMountID) +} - if resourceFilters, ok := filters[collaboration.Filter_TYPE_RESOURCE_ID]; ok { - for _, f := range resourceFilters { - // For shares inside project spaces, if the user is an admin, we try to list all shares created by other admins - if strings.HasPrefix(f.GetResourceId().GetStorageId(), projectInstancesPrefix) { - res, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: f.GetResourceId()}}) - if err != nil || res.Status.Code != rpc.Code_CODE_OK { - continue - } +func (m *mgr) getReceivedByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.ReceivedShare, error) { + user := appctx.ContextMustGetUser(ctx) + uid := user.Username - // The path will look like /eos/project/c/cernbox, we need to extract the project name - parts := strings.SplitN(res.Info.Path, "/", 6) - if len(parts) < 5 { - continue - } + shareType, shareWith, err := m.formatGrantee(ctx, key.Grantee) + if err != nil { + return nil, err + } + params := []interface{}{uid, formatUserID(key.Owner), key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, shareWith} + for _, v := range user.Groups { + params = append(params, v) + } - adminGroup := projectSpaceGroupsPrefix + parts[4] + projectSpaceAdminGroupsSuffix - for _, g := range user.Groups { - if g == adminGroup { - // User belongs to the admin group, list all shares for the resource - - // TODO: this only works if shares for a single project are requested. - // If shares for multiple projects are requested, then we're not checking if the - // user is an admin for all of those. We can append the query ` or uid_owner=?` - // for all the project owners, which works fine for new reva - // but won't work for revaold since there, we store the uid of the share creator as uid_owner. - // For this to work across the two versions, this change would have to be made in revaold - // but it won't be straightforward as there, the storage provider doesn't return the - // resource owners. - return "", []interface{}{}, nil - } - } - } - } + s := DBShare{} + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(item_source, '') as item_source, ts.id, stime, permissions, share_type, accepted FROM oc_share ts WHERE uid_owner=? AND item_source=? AND share_type=? AND share_with=? " + if len(user.Groups) > 0 { + query += "AND (share_with=? OR share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + "))" + } else { + query += "AND (share_with=?)" } - return query, params, nil + if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.ItemSource, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { + if err == sql.ErrNoRows { + return nil, errtypes.NotFound(key.String()) + } + return nil, err + } + return m.convertToCS3ReceivedShare(ctx, s, m.storageMountID) } func granteeTypeToShareType(granteeType provider.GranteeType) int { @@ -574,48 +538,49 @@ func granteeTypeToShareType(granteeType provider.GranteeType) int { } // translateFilters translates the filters to sql queries. -func translateFilters(filters map[collaboration.Filter_Type][]*collaboration.Filter) (string, []interface{}, error) { +func translateFilters(filters []*collaboration.Filter) (string, []interface{}, error) { var ( filterQuery string params []interface{} ) + groupedFilters := share.GroupFiltersByType(filters) // If multiple filters of the same type are passed to this function, they need to be combined with the `OR` operator. // That is why the filters got grouped by type. // For every given filter type, iterate over the filters and if there are more than one combine them. // Combine the different filter types using `AND` var filterCounter = 0 - for filterType, currFilters := range filters { + for filterType, filters := range groupedFilters { switch filterType { case collaboration.Filter_TYPE_RESOURCE_ID: filterQuery += "(" - for i, f := range currFilters { - filterQuery += "(fileid_prefix =? AND item_source=?)" - params = append(params, f.GetResourceId().StorageId, f.GetResourceId().OpaqueId) + for i, f := range filters { + filterQuery += "item_source=?" + params = append(params, f.GetResourceId().OpaqueId) - if i != len(currFilters)-1 { + if i != len(filters)-1 { filterQuery += " OR " } } filterQuery += ")" case collaboration.Filter_TYPE_GRANTEE_TYPE: filterQuery += "(" - for i, f := range currFilters { + for i, f := range filters { filterQuery += "share_type=?" params = append(params, granteeTypeToShareType(f.GetGranteeType())) - if i != len(currFilters)-1 { + if i != len(filters)-1 { filterQuery += " OR " } } filterQuery += ")" case collaboration.Filter_TYPE_EXCLUDE_DENIALS: // TODO this may change once the mapping of permission to share types is completed (cf. pkg/cbox/utils/conversions.go) - filterQuery += "(permissions > 0)" + filterQuery += "permissions > 0" default: return "", nil, fmt.Errorf("filter type is not supported") } - if filterCounter != len(filters)-1 { + if filterCounter != len(groupedFilters)-1 { filterQuery += " AND " } filterCounter++ diff --git a/pkg/share/manager/sql/sql_suite_test.go b/pkg/share/manager/sql/sql_suite_test.go new file mode 100644 index 0000000000..e3c6fbd8e9 --- /dev/null +++ b/pkg/share/manager/sql/sql_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sql_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestSql(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Sql Suite") +} diff --git a/pkg/share/manager/sql/sql_test.go b/pkg/share/manager/sql/sql_test.go index d9d7c414f6..176e75eb66 100644 --- a/pkg/share/manager/sql/sql_test.go +++ b/pkg/share/manager/sql/sql_test.go @@ -26,10 +26,10 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ruser "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/share" sqlmanager "github.com/cs3org/reva/pkg/share/manager/sql" - mocks "github.com/cs3org/reva/pkg/share/mocks" + mocks "github.com/cs3org/reva/pkg/share/manager/sql/mocks" _ "github.com/mattn/go-sqlite3" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -44,7 +44,7 @@ var _ = Describe("SQL manager", func() { testDBFile *os.File loginAs = func(user *user.User) { - ctx = ruser.ContextSetUser(context.Background(), user) + ctx = appctx.ContextSetUser(context.Background(), user) } admin = &user.User{ Id: &user.UserId{ diff --git a/pkg/share/manager/sql/test.db b/pkg/share/manager/sql/test.db new file mode 100644 index 0000000000000000000000000000000000000000..fba76fdcc4fc7816bf2b281f9c2074bf1955842c GIT binary patch literal 655360 zcmeFa3w#^*b?1v0i8n^FM8T9q36e}xuqYCPH>AiGMN+mbiL_%kfpSg^n+z^=KRl`|M`Fa=l`EMe*iPjJ$p`+g;+&u);Kwqa%^)rosN^S zn8V?4(*Hj||Ld<7ZFuw@`m2<0^l~~T&VJ_rY5ADnQgpwc`5N=L%$J$JU_Q(I3G)Y< za3doq2LTWO0T2KI5C8!X009sH0T2KI5Lk=AYpa7)Bq~|^g;(h9sVC_zeS+Q&j?>%D zVS3xagz24jC;hd2$r~}uruO5s+86xS*(aWCyP?OeM#(cDFmO8sKGdS(I0bwQGz^Yd_Z!1sSI<;ML~2l>Fbte3OIu8G6GD1V8`;KmY_l z00ck)1V8`;KmY_l;5G>qT|q}?WMstQJ9@$|i1n6CZzqKtT&+u9U7jq{&ZM%+(3;fsr=3BEBN~l5JV$eWu3QuAmN)PFhBmUJY6qxQ zF6Pf2%vb0QFAx9$5C8!X009sH0T2KI5C8!X0D*g)K#0D6pf4S`eD0{WWPtqt9`9DT z4+0js*?sDdPRr`oF7*#$ z)NgdEzeW}7i&}Z{x+pIdODiwE=^&dvcYzfth zoo{-R?$EsfzvA8~eXn&iMee<%vK!r}xKret0w)U^LDR{ObPdOD8&(I?exKpT`SA9P z=Ettv@gt>no^-uFU6tD9*mJ#6sK$oHrbu3hvC(v*YNPA@b)P!%5KZU zr0U@_OOh_UZ|;~cjz`oOUb-UZ+;U>LOo$<;}!Rs19tdTD8_eCI5FaA9m0`yg&d1KmY_l00ck)1V8`;KmY_l00jP}30!je9UjlH zvZSmmCUf2oaoW1DzV52#^S8Z}tmOZ0CI9~n^UGKR_?PZv)DHq600JNY0w4eaAOHd& z00JQJz!Mm84?1#Azf->};4wY#pIGKAZr5IyTI{zic0KGIbg+KES*m_}N;y)y|Nj*S z^L@;f2c9tifB*=900@8p2!H?xfB*=900@A<9TPa}@;VCiH~@d6Db{(h!Btz6$`7hq zlWHI|au@zGuN1*)5$D`f<@24HiA22_C$4!I*5C8!X009sH0T2KI5C8!X z009sHfxAoKkf$(WT^q10CFn~Cu7jQv!`9^noAm?b^Z&Dh6AtFjm|tK%$b1{qV&)l^ z8H#)%@@tWQA9*dZ5SfYW3;(b1$HE^Bza!iTp9}8|{Z;69LLUizN2n2cDRe9p3%P>7 z9sFqUyMv3tv%&G8Bk-BPuLS;W;4Ohy0&HNH|EvCA_y2qUd;Glrr2nArtG++-{fh5H zzHjvjzB%6sU(EXz?{9m5%KL8bi{8h*4-fvs;2#eDUxV)-Tp3i{!V3gI00cl_OAy#| z*26}_?FK!KR;bgBmntI9$s(;-uQY_#_|qPCGTL9Lt~FPyLbqs99!XKQ)nfc-nJd&kj?|w4FTVIU0SU z(Ug`&rO9Nm$TdW@hDK9Zrh6L|7n@SMQEUn=xhe7r#mA;RM<=7b$Jo@InMmwA={dS* zN3X6~|Hu=bvFILaA9Xcv>~YW7WVEZ;ww~mk6P~d>JGvCjN|-lzhNIyc*SspoYSV^} zdxnRjnm~I`Ok}CAk9meCJ7Sw^vJ;8@1<&x&j)wL_Mp?K)w|Z0hJUNk?VE5-e!$*cS zo04X(fibHO&eRq^N=enn#`R)v#SDS8Q;;)vN1D!9kSl_2dc>Z?TQ zw8Xw6N|5&In$^S}qaf|TQ+~`!qeQL4(x5oS>9A&@qtRCt^*;^uJ%^P*?b0+WdU(up zL~oA~H(GNZ^&HVFFiPyIpja3g^&HWwFm%mIc0b~o)GMh`=yFHP;ilX^WCWwWLO2h2!NhDo!|ReQn3%t+8x`f5u|)J+VT?3n4Oc&Bg4wnY^wCtP60tv^TW#2Y-*VeYLZrkP*!WU4Zew8 zo{UjZm!eITG$$V%@??xEyY$Vq3bX=$~=}P_bT-CYRYstxPMLXe>3xw|Ry}Pgsspo8L8T zI>#v5k1$ln5ktzu0o3rel~ z1D??+`BATki$sO47J48*>i3LJMy(2lQ+j2}wMk;P&ojDbhgHwSzSrv+i5}vmntFdi zwSvwi!#JToIOrKsYS2n;tkbNxYrr$2Sl4t+%ey@^gKh6>n?jwM$maIAsn!GZ{Qz;9 zE+TrGb3VwY2?RM#;pRX+be>-{SS^B2s2Wj@CI0IdL2naj*cW`r4x zd@k~G{D0;DAO2tP zf5iWN{ZJu3oj4=0T2KI5CDO@Ltrnh;W+yj zUet?q-yzUrYGPFM`<1$~AmErxZaj^PP4l6rBf zrjM$T)LYbNd|ZvBUbT+-BWfgRh1P|V$J9vblJ^;otC7^D)3JJZm{vucT8Yums?TGz z(lOav(r5Y6J+vOutJSeRx`!4+I%SqM4AuH0W3(93U)E>;&BO1LG zU47+d*yr)6KVDbgmXmKzX9OWxsbn$&UrLs9g(RO$l?tUozEa^xz`Z$F$(4k1rkqLT zlj$;>Or%*ZQA%W!rF=GD7IJS+r7NjQk`)RiE}70Ia%>@+re~Vwl5A4Qb9t_u&14ev z?Sm94RD^7)T;UU3flDXYR7R-eg$&16(%HPgB@0}xP);TC<#MW!&2t=EN#-l1G%v8Z zLONSwOG%-^R?-!=oKFk%%lRyyP4cPp(yL;%%1zSOIbw%jWZC1fb7H-HBX%R7EoRg4 z*i@rY6l50th8k0(=b9}nYlVdDb;WIflT_(5qLb=2h3Yh{YRI+R?ndVYi zHlHZ4*;KN?XA^89EeNcT%(1ydC0*tcxpX?qrE*l7OIFx)CPyUsLZOgKaG6Rn$7O^< zCXvsjIW}L)=jj;&AE*C!DWCs4nO7b34=)e^0T2KI5C8!X009sH0T2KI5CDO@PGHsJ zxf%BVORobcpZ`0y-Sx)7AP9f}2!H?xfB*=900@8p2!H?xfWQ_afc$?8bp~7k0T2KI z5C8!X009sH0T2KI5CDO15CP@;|M>j>8w3|-K>!3m00ck)1V8`;KmY_l00cl_ix3D$ z{?NfV{SGD{`An!8`Cz2s|9v0p4Gg?@;04cBPtx^@>k%sc1^VxGy{5MhMqgl^4zXSq zZnUmdMOi4)<2)pFzbNrVo~sJ=GS`$!S6){IbWr3E2T#A=V~yi(KFoNdEbF}WlB_&{ z%HkQn#MKvtR=;H1^o7}}i?gwdQ>V_&#uoY&7Gj43vDkuGUWn17dW1!x8H>%$UyRLN zI(IJi%!M=Or!HKMot?cLo4Ryy{>&ViJ3l*jF)&WYw435WY?*6P<;}y%OeP+inLRyq z>6~`3s?u7i^RjeRsM}fBwOLhFhivskcSKc?8lC5@n6`}acrG#+Jv+KiA1cphQ9UWL z1i{#2Ze0j_qobqFZ&}ot8f8|!NFPT-?M?_PjnxS%x9-pzWRM&iQ`=6D?-1+7I#&}+ zeU$=n=^HqtSSVXsvHB2f!s~XkJ8RfbuwXD-=x^KxuVr}P?8+)34(%*)J z)&#mUYEi$l=O)$pCAw9~hPs#L&OCc*Hgz&neJ{K!2GwY| zHDpACUZzBYz8&h12EC{g4a(zfEPb0wu8mJ~g2!c1S`MuF2N1SgTHc(lL6kF3m)>TkwzAOTn_@$GD4yAIcRC9iq1@H3 zan&E{vYU_iywP0FdCQ!vOpN9^z(T$w^smRiqjLwnunG9LaVD_1|N2e_w+?%~(RkeX zEkPqRJ5MDl)~+d$Y3}*@!I*^|A~jB|7VSLQ;QYwdt`%jmMh{QqY7KK1)8|=ZLe}O- zMO%4@B+Zg7y4u*3zBcn3leBUQ)~%~^u40p`R0N(LQ>Z+A*$hKB;$obA%Z&z9R$-8DJ>9eigXRo7|Hs3=u zCZ#4l+G@kDvFVon8K$p0`cyXCa}5fvwPRot{nb&?`^q?}bj1?Iaf@5*z5eKPx}(-- zt4~q6pStO{78iX=Rk2Rzf4y;QDA8p(1!QqE)H|xSVWL-_2*hu_$*o4^x1KPglBZ|6 z$;uTr$7`D&o2b-E?qDVrl*z_yvHkf(8OrK}q=V@nf>nv0I$GR>V&|l2cWtQy(XS~M zuC*nZGo4c|YWj4PTfZ08%he5siE_($b*9ss(KjhA(GBX%`q)I$a*3=?NHn#|q@ian zdcYcPsZ*IW(EeMXAU{n&Ik1fC?+l8FqrXd_AoUA=|s^0Wp_R$ z!+XXxs9a86RwJ&Hg|>uiZK2K!J-hC%$m=OQtX$uW(~QkykP zC~Nz!E~`qa*lDL$dbkfoywtmQ-&4+p;A5pcf6<-6qDHt@fDf6Mht z1E-yDiF`P8GVnzz`oZA83w)xJBM08^ywKVFXZ%A>{Qf^+Sj;P0YrNPG?FFSadfF$=O3GQQRFz7`*f*xFOSgySKO$qc2ZZl48C%~5Ys%1tp!#w`ln|FGnCr&uuXHS3y-NA$5$fRyW+|s;b0QIhtlS)RJY6 zUs9jcx{m4^0jiIRt`k~UCmFfp{Lo;OKjB>atfo`w3MV!1w=H@2wG$7iYbEb6PA`TY zojZ)}p|#`o4q~;djT;1_ZHU=-?k0x7Z6va{u|%Nb`bI>olHt56)oIWeSN44YOHTo6 zkP2Pln$ug-ZfMa%#p*4(x+hgLT=g*Q9#_T{R##^k0h!ye!6Kc`L|c3^_IBOQdOalu z-G^%#KF;38^O~{BuN-BG;m$GJ*No(j!X3o3-nzSRF4Ij}&z{E2r(QL$ zF`4uQ#T}gOl+Oo@nCi~V@-|bu`aCR&=i3xl$~i-ySdD-(PUed5n^J5m?bkC4`_aym zu@;^4dG#^wX01Ai?(D(B)revBQta8gdO>z`eptOsc+CvV&J((dO}YnoQGI-3Qw>i$ z`QH8@wNvYas-6)UpsXpF0l)JhgjRErLR3(ul!chSWhffG)XB5EJAOUeodc|WwvM_n zv*`YhFIa0d_Q)LNbFWn%*4CJI7(=~-L>c3q%{^;!yMs}!&68bghsH=WplR12vec+a zlsezWA^eh9r4{-*wSkr<%p))6^f%ch3+9#Q1p;K? z$1&^IX|X~{xuQu{M4)#=kNIaU&z@=}>wY0)=Q6+zJu96C*}=_#ve<@f(C=Eoe&|3hzh zfdB}A00@8p2!H?xfB*=900@8p2y6iYhdhN5r~NX5zI@<1>Nzp#x9D`O5%ety;Qs$x zpj+Vr2!H?xfB*=900@8p2!H?xfB*=5g9+gN|KDK1Fb@JC00JNY0w4eaAOHd&00JNY z0$YH9`uqRLjDyM18(ts)0w4eaAOHd&00JNY0w4eaAg~UB)rQOO@ZS`2p7X9#vrnFx zJNEqa)PX-{*tMAvMSLnM3?9$kH_+fd?GM=e(KCQflm}F zd?Ay{3t7iUS5MPJ@%%TV?A>azdMf~>z1l|Sucf7`ysw%4ZBg|)4GD9lL?y|mletnl z$(G9LOgWX!m-G2Th0Pbz>73&|O8)O>4mg-EF@MDT8uRa%?_pM%E6fau@B#r4009sH z0T2KI5C8!X009sHfd_}cfw0%%cSbWlm%65)FHKmN6dnnwD#v{;by31l>0Xl<3MyL6 zsLwU0YbfdcLjgs+eZ=QFZOC`{71<7w(Q1gU9P^OSPmZXS*ewTacPkpqe&3v0s#uS> z6w&sm@3bakoXY-=sE-V*O8EW%gA>YVDF}c72!H?xfB*=900@8p2!OzYO91)*gWLCL zIS7CN2!H?xfB*=900@8p2!OzYLjd>xdvN+1Ed>D(009sH0T2KI5C8!X009tqa0%e^ z{|C44(Q*&~0T2KI5C8!X009sH0T2Lz2ZsPY|9^1$8Z89@5C8!X009sH0T2KI5C8!X zcyI~e^Zy68@6mD)009sH0T2KI5C8!X009sHfd_{GKL3Ak`Wh_-0T2KI5C8!X009sH z0T2KI5O{D2D4+lDaDB|-ey{6e%+E(Y9(h~%c<`5k?+#P~!~US}6W;5Ci-WO&^1!g? zp!>6qA8|NBZ~Ygf<~J6!TWGh0=0a?lYw}B6^KdpDkIl_rjLls- zcP=na#iDAtBCe}sabO`l2icNtl7i(Pes?cl|*|kbW3gd&( z^JD86tcX>iNDnN%s#!626RYwOZ**+Td23APY!*^|=FVgvWwWdUSu8Ka#JVgj3eA|o zH)aJgb?M^#nK`m_es=DnN?viwJWB5rgK9!mx+<)!(|NL7UB|*vRxxpD?##27W~oCI zQ`7}#9vh6ll3R}$1zVGog);R)i(*FTkn0mqtQO+lXfEfxx!oYxr=*bV6CUj++NaYY zt9nqh^3_SHe40W{q8L)j^$<^HGTkw55HM^JhS|e|(aWiIFi?aG%{o^tN)1_*>KcSz z;rPw)m^YeAId4tr2zpfrK(FXXU-Nr4I*r$=wr%|e!Bvymb=mg7O@!4lS8WScpxHHm z8nuk;(RK6OquwaXI$xXa%hcKG769eWLBy#@VJiWzBKD z{=C-{py@S#Vm|RseBDK3btqbcZL=q78pCy7Y|w<*6wS#)J;n5rSkORrord~! zIGz_;Epw>%DbVRwYBzbo)>(D#ZnR4jmK*zH^h{S9#uqF#jB7rRFPJu|yKYYH_eLit zov*Q0=8dM*5Sp?`{%PDz!694rX1NW2VNCr4UgvMN}a29 zox>~_1x66}9MTi+eHRi7x;yqm#xTnU;#2zuqn9SvnPrsdydsr~Et!+s+Qmh;=)me* zN7Qb(b;Q~Y<}saaFab$-N;s+sM52aSxFE4zSCu=iDJDXnSGc} zBof^xU?X1D+9(c>?;VWJ$Jb$KNlUa8!OWBPby75C8!X009sH z0T2KI5C8!X0D-$t0H6QgeWtJo0w4eaAOHd&00JNY0w4eaAOHeejQ~FX-)cPqe?R~P zKmY_l00ck)1V8`;KmY_l;O-N^=l^$~DJ+5j2!H?xfB*=900@8p2!H?xfWTHGfY1N8 zT93dV5C8!X009sH0T2KI5C8!X009uV`vma$|J`Q_iy#03AOHd&00JNY0w4eaAOHd& zu+<14|KDmo0)IdN1V8`;KmY_l00ck)1V8`;K;Z5Z!1w>}K2ul(0T2KI5C8!X009sH z0T2KI5CDO#MgX7xZ?ztQKOg`CAOHd&00JNY0w4eaAOHd&aQ6w|^Z&cg6c#}M1V8`; zKmY_l00ck)1V8`;Kwzs8z~}#4tw-Pw2!H?xfB*=900@8p2!H?xfB*>GeFFIW|L!w| zMGyc15C8!X009sH0T2KI5C8!X*lGln&;KL7{SM|M%r`~;D)NEIGvTj>zbo{C&}{H> zV9_u6-b{sffdB}A00@8p2s}sxUW#mY_(#t>hn;?($K&&jIl~pPDzqkBOI%YZ3x!I7 zPgg4CawVNBr-VwrlF8(<4>|l# zoOh<2PPbzHymR$A&AfEIK3$dC<=A59PScp;U` zm)X*SK4ONm@I zDWnQ%E}6?_$jN*`;8{MIOJ%iomy+2sH8L$^)5%mSAtdveJeMx#Gs!$(ViWmtxx{k? zA*HsPP3PE5w!{}Q zk}NK@Ii9O)jZT*moKRv*iEJjBOJ-69A)iaK`LZBnDml*Iqdz#{e#=Mo7IR#dV}*iH zN|q{#L?WB0a1=dkAz$V=Ays0v7UwykkfBaVvq>(IC<)0#wm@A@0Vd>y44>mud?J(0 z=Cq#9vJ`O?38f@;2FI5ZJe%i*icl$4vZ-vsXmNp}jND~c#ef#vqiePJ8L26^7b&=@ zj|&AF7;GY0q25X534bM#OOn&+Oo307*p%O@Ol|$CA-%1sawgBGDB@BXmgCCVf*_=M z8YyKipCEgQPFu-ZndeG;DJ>+j)I-@MVNE3XL^;Qn*kqYnk}OwPwXF%Zl*;oo$SMNI zr&v;?0iEXxX@Skqz$qDR%@#6+R3=}3qu-b%)NyRg5?V}`$zUlGRG*eJp1rzFiaQa%*z-23KoTg$b!F z^Z5dw$tGyXQg0@?bSjg}r?_012qoz~;kD>`+>ZC$qG}bOLp-@tN>|b}%=kRTe1hl8 z$y7O==Q%D@;wla$8IPcsPF=DM{Nrz6V$S~UrQ70RXj{uf+S&7SlAJ(g;?WlrXrEl%JIbmFP-8S2DL zsZ>s6Qk7(q-f7seLZYLl7!E7{cMO-y*{mS!KkFpJ8VzUXob1{d$JcwkOeWJD7gaw0 zkA(l+5&3K8B(pp6awHqM8QB*3^~i^qO8C!dRp8ys?>G`vgWk{`1a_7HlJ`)<#RMVO49Rb;(diYyE}1psKae`d2{J+GrukuWD_y`r}iz zHd;jSs#+T@hYV`AHdx#kP_;J7(LIlPoaddcjTRr=n%YKd2(D4G=HGB(0Kfmg_eVBb z00JNY0w4eaAOHd&00JNY0w4ea8xaTx79HE2?{S0&0~5ZFcz?#XJowArZyp>T{2tFw zx_`s-X4mhyzt{O5*AEB&V_0xhNcoK$S?nqIDf_M;c8>MPvD;7J=C0%3=>GlATW^xN zQdN-nVohq(cbkljlhYSwr!LOME>4{~Hyc|pcZvN+1bmnsY@5<&&-jL^Rsgo1LJf^OO~44qENIuR8?q7lDu|lyCpQ$ z(q*p6(^o6*{86^9Gxi>t&s$bB%`C1U8p5o1k{>ve0aiS&K?f=UYSX z@PcA(d_g~m+E~aBM$eC}+eY1?P9yc*6E}-lZ*viS=*IZP;kZ7uR2YXNFFOX@u*lZI;edOxVkx(J4<@5zQ{;FUZ z-Q7A(frjC-P%d&b#_Y`qHw3<|1VORf)}q1GG^PsO>QOcje>JU6MXS>~;KioYZYYSg zjY1z}P2R!0s7xq2Fzcky)Bf%dQI1lEduD^-u2rQ5g|>NYHIvlBKHlYTQJGG3pVgh5 zz29otZTh~@e%*~6SC3W3Kx%_Apteu-&TOu_n_tZ*ywP~vdGknjleuzDtQ&1s_cCiV zS}pt+X!UV+FghP!XXI>?Y8Wt`r2(N?6g}s zkJz5zHkcHweNVq=Fz3de2v^T&n;HuXGR@-jz5kkZ)=jQ$xGM4LylxuLumt93S1i!I%*Pu_h~ zvUW0%N>>D4E_U~1pS_NjH9-4xT5Z+TI$m`&iVd#G(S;M8>lb2GslF)Fx1?=CM`*R^ zIM>=6 zZf8PKgPQG&B}1(fN6Nm!y{{jw=eaen!flSz+pG0|gOiswB%&x4muN0DHLVe8^qqF~ zxj_9(zn|E!>N*91X;7W;bl4=pzP+OO+s?NF|CsosriZHcc^ts7ZW2GE9DdZts` zl(takn=1|3dd;=Qm~7V#-QuR)6=ZJFB$=i&hYrU8*$|pFv87yG)1cE^uWM-t)l0%w zyJlMIpRcXSMW^|61#ZR%U4P*7|JQXA!a*Pa0w4eaAOHd&00JNY0w4eaAaG9+!2SR4 zsqTWaAOHd&00JNY0w4eaAOHd&00JQJx(Fzr|2vs4JLn%?AOHd&00JNY0w4eaAOHd& z00JNY0{07nu+trlsz0Pt-;~e)k^kQ>or`vW00@8p2!H?xfB*=900@8p2!KF`K*SS` z`px`b`~3ea4(2N$1Ogxc0w4eaAOHd&00JNY0w4eaAaI`%@H$=EmjJZS|B?USXZ?%j zfB*=900@8p2!H?xfB*=900@A9Lcr(lddtgk;Ogzh(^9G4u2wnk`#!GxKfBH6kW?e{(^D^B zYLq!yn6o^0@#U_aQyhO)d4y-ZY?_HEW^#_oulD6VnXj8LU# zG|o30OI%$j+aCUy-{&ZphYMvvdl2x;&#lz?9uJ@J`5e5VJ4Me}To&aOTSlEiQD|8~ zo%C8!uVtXdiB)UhbS#9GP$>-h94{Cy%uGGsQ@a|c&om`^R-SBDt{!@1z~?yLb?7wL zEGs6uo2n|G_V^sv3}rPa>}qN^o>9WBCDY(hl+H=~RlUWkoj16Bj#oBg$M)zaTvp(A zUUc{H8Cj@xRiAP8#Z*r-l(19{TPZ8~|CC>FFn`MYD)ZyacQY%@8<{7W1T#!)0iTb2 zD)MuYAC7!`Ukv?t z==(yqLbcEfp{dZ3&`|K}!OsQ%-{8*$-y4*IPX)(<-oR%AzZCdj;9CQWfhPlFfkFSD z`hU^?0sq^{1-w821V8`;KmY_l;Qk@7Z^}Iuw?1ONl}xbtL^hFTQ^`y+x$mTHIm;Fj z`9vX=+V_M_d5+3hHbcj!_dUK=c_N=lr4##4xa0APnls6rbcY;J45aCAGLzc>hV>Lk zW8ZPAL9@`((w$@?nMmZ5DK<%MNbWmkTb`i`ST>tU?<@3|Q|r>nESpTFs0DeO@*LUD zu!&SQliQcGEvNRgnRFqM*q5~{PvldCD49xR_GRqLbpZR)>y%Ub_ooa%LMP4B2(sx! zhD|5aqe*urUeR(Yp{$b9IVtUA6Kpn-OJ+vdwbd;BF`C#|jl9}9>E019bIWR9q?0*v zB$H-$PFSQYn`ZNDI+NacRF^7IMNEilGQD%$kmgw>nh19~z4M4EO=OeFL?Kg1vO6C$ zq`5>PnNvb1o!J?;NONSPkRnfZ9=1p`)XNF7o7_2Om69I?R2 zY9M)$VUs%_>5;OjLL#&CkVQ)IMxmWa<#!(Jk!BR>&;j>QoR^w{3XL3N*+ibr4aE#G z`9*BmY?AyN+HZ>UEcrkKDxJv_AW7XP9i}; zHnhzuR$?@h&SnZjj8&Xc##pM5$qYrh#5C%2seCds6t;?!G}RE5RCXxTD`wN_RBkBP zC8mL)#BY8m&@E0WlTJD_5l0U=SfXurTcIEu#J=YneoD*wtN>rRLGdRgZGo6{)8N z<@^6r{!u0W{}tv(neSxU%w=YZi8I?He;4_mkza}YhsgIuRwH8M*+?OBAmR^yIs7}} z|3caScZOHOT=-0w4G)K%q0fbWBlNM*4~5FHfdB}A00@8p2!O!WArSM7 z#hvTq?^A0i7zzAITLnu7|HRq~S_=R80Z%;cTrY<|vAzbSG;dHVStpA>zJ`L4#vij& zQ1V+nk1yCNSQ7dC8VW`xpR-pmQ~BVJS|SyzELq%EZEv-#ARCquM1Oy`s9=~?o5 zb~Ae9Sz?!GN8H($(NA`Xjg)@E6kBupqZY9xsUNqB&8+^2MQlmyAL|kud40S~Y$W!F zyTy8DKh`BSQu{}{#71sE+9y_%`$xLOMs|OwPpqc*2cw>$xU)aMKVXqt6a1J}ZprZX zcgu|we_yxU$ni(|-qlnZn=^0Z|jyD89&n_*HivTx7^72!~JqK=@0eD z^{hYGEAOQJfo{2x_xpR~dgAZvlWUp3caLWz?zB(+2W`sq+<&0A%#!?j`pb;$-`!hg zN&j8D$rr!H7p+Xq|CP`Gy^QQ&ewcX=^S_v%U~Vxo^ZQ%p)C*5Q00ck)1V8`;KmY_l z00ck)1VG?kB@iBTI}V67Zc&)L(hwGX18zrD-Hp23j?9uQH_DUkx_IMgnSKev%Tja2 z=Oh)i;4zQe@eB2!H?xfB*=900@8p2!H?xfB*>GDS?Rl zfYZ$X2mS7-wma~!JL9y=|6R)c`qY9W?w6dF^nJbb-?PJgq)V65e=p-cWRKs8-?RR00ck)1V8`;KmY_l00ck)1V8`;wg7=0&OwLY8QtUI$~Cb*sl98@|N9#U z^EX?-L%09}AOHd&00JNY0w4eaAOHd&00JOz_Xq?$QKw(c|2@j*|L#c5!L%ZON`LSI z0T2KI5C8!X009sH0T2KI5CDNYC$Q{}IOs-(!_lML!`t7oJzP_t|EMOS^u)i2wwsv9 zPOxqD0h2{;nH~_xm8!x^ue|h%^8NQ?lap16=c-FmOFovT2bfHX^|EkdqOsJNq_4%d zCN&$Ahm^g}&P%VX23?F&@lmzn)$M&XckZjI+zhlV(hu&F6REr9L$7J_Cg4=PIWnwZ zSY_7mhQ6)8EDG0!W@3U(P9*OR5zQOJ^h>WO`M+b^op&uPfB*=900@8p2!H?xfB*=9 z00@8p2y9&f%J=_~|8L#?foC890w4eaAOHd&00JNY0w4eaAaFMbD4+kkn4fhpzejI) zfdB}A00@8p2!H?xfB*=900@8p2y8h5Bd&pHe27thKT#2zExFmQUl(8Hn&sA#bUks% zIS?Hh8B>dSsVT@}O?XwR3$yJeJt}bWl+dh-b>;Ja7xPI6^ErCM3j{y_1V8`;KmY_l z00ck)1V8`;KwzsA7*SROoNKHGxOORv0oqOftcwBo{r^_)R`>@3AOHd&00JNY0w4ea zAOHd&00MVTARJzC1e`zR2>evwn*V8k*!#zWKR0mR`L*yzN%)iDw>p;Tzq|eluQ=a+ zM@Gqj^NtMM>jzIgIT*b(>2#>KzieHrin37T+OnkX7bU*fTH=~QQMsdKv(8nEY@g`- zx6IFaqa!2EcVCm0TWsnI`rotD7iOm}&c-fIojNxgThPS|vBQB_Y(Xq9#KgKREDFuo z-2BDZ+@*8pV$WPSbAIZ=<=ENT%dx3T7w6B+k&W}Sa~A{Sbcp7WywVVC4xO1jJ$32a z#aP0kdR>&47Gld>lc%;EPG&Okn69Q&r)svva#6Zo7n-JGHXS!z(hnFLGt|YpD2p83 z<+J+`UDGh#Ab@q*^tDfcnAeJ0dvqt)t6NoXN$n<20Q3XwTcug%nu`L}zrLz+N6(^e zRqv5LS5+;!$6$s{+7N9B&6?P1iBi2~w%zDr-Eya&l7Y;|kJZ`NkK(M-nq+FOhf#V-lvHhs@(Q7T;#c)6tvqh8t5{Ufbc#b(r* zF~0t=Yc$0=FE+SpyD6GYw2wQwvo0~CFOYFEr?s+Fl}g>MREF{gxpStEuvun<)`<4? z$tr`&IuDwjDU3QbtaOK#%H6Q92U%x!(hpgt&a#AK*D>lerYmkRnX-iFu~Xh?Jnnpr zF%gN&G`5s3Rv8(4Puf5-3Okgp3M;lP-eBr)w>CRL(`}p0Xy|IV5jq`MnnFdOvjQ)))HAAO|MbmAPHLpxCTT-kC2j1@+K@I1JERq43To?| zv{u{@G^?sIh}Kf&rFunN>|(dxIZIRT0JNX=DAujRo3&Uhd``@~@M za%w%sLbD~dWIFX0=~mxYJD_`nGjD(Vac?x0a=z=3LAysqLEa;p?&sd4VKW(+D6P%x zzr0aTvDU$%cY!)$)oYX@oe8BjHO#7JM4)k|>A9X^6A7Esk#gph+Lp@`!%ww7z9}e! zyvVJo(Q_JQ6G_W;lWIfNC@yg=%T<^)wp67SPcv@+nO!X_(`lcM)vrpdKrYRldG^w5 z?C^r6lDKlv^!(g&7cWemnY$QUIDNKwIx(G^$z)CyQq!k$h3xb~?DYJF*)vbhsTY}t z7j)|J*oE2CvlnLPre~j1FIJTTo}=?9b*ZXF%;T~7Il_HzmM&|jr=FXhnwedv%IQLa zoz9*<-CJc}SN7H2>1Mi6IILVK+f^H<1mq2`Hwk&|sng!*-fHes)KR@dO}kNT!ilVr=eO1-Akk5z*}(@;ZRpUc*m*N?K=MY`r&PkdTuDb;R+ zYqhRR&7LVzKcgChX`R!rYNltf^;NU>sY^Y)ri|wXrCpW_^>jiRv0V^MwT;HprZ1M2 z3#=B;)y?tMHQ^?rcbO#!89S!lB9$fWuM zi}HTX`@U!J`Ty4KSa=2kAOHd&00JNY0w4eaAOHd&00MW5fb#r5$F{p=4R$~P1V8`; zKmY_l00ck)1V8`;KmY``A_3*|e;4!L9n2T$4KEM?0T2KI5C8!X009sH0T2KI5CDOD zg1}zaz;TD)K~GNMs>hB^*6Q-4@}!?h<#8CT$!*SoLsYsX)r4cm)E@;XpZ~j<-*hlv zrZ>Dm00ck)1V8`;KmY_l00ck)1V8`;?i~Vq+{c}3tN^$o?n8b{{*T}P-#gt3PeA|# zKmY_l00ck)1V8`;KmY_l;Pn!~@Bd#fIUEN9AOHd&00JNY0w4eaAOHd&00Q?80p;`m zT~5~F`We@2kq<;7@y*SudJ{PN)cIq(kyFL{2=Gw=RcmxGkA zI#$Pv-l)Uj{^r|j=_hVaeJg6Oo_oa`m2%G2w~6(#a6>HLC`x>hE0>!>t5uSutNapI zUldwakytLaR_eSgT@~uK%H`<`vr`vmV`t`OW?zgQyltZgWAk&dgSMs*#ttu7yc5d{ z;|qo}3-MLw%iiddap&rOYjf0JixlN1*IFvJTSBv_7Lt8S0dMQGMZt2U;h&{gaZItj z5YL<$jGj5-E3RBjyGr#)Qr@YZahn%;jWv)~eEJc>h zxsI0?r%s)ljV)M87Gj43vDg9ui7j(Y>b2(KY&ssBnLRyq>D8jv$L0DQNKRb6(J(OGK#H!+(&5?RN ztg2b%T5^$J68Nivr5asF_vO;unP)G}5;yWC9*D1Qd&wJpmijKWR^Q2DO=!tntzp-J zr42i<=kT?9(NJ56zv-L2(P{E1x|T;Y6x1T>2umz*8@i$&x0VkIk=4*;Z}ifrbM=U| zcCOLjrFunNEOu>?!K=awxnW2bY@INQO+{r(JKXTb&{Bt+QcUjbd(j)^$(^Z<-0Ap7 z{upo!gT+mI)oaDxIZcbIA_qrb@J3&mbgrJYI#}kGi;bq#5Sp?`L$0&A(%|GJ!@F*s zH`=<`tz%=y@UL4_A!x01A%6Dx!Dz9t&it^*eLTd8xkp5E4d*UttE4X)L?E;TI>*KBNGR4DFH|4%oj^~FW%brj5o@%&bOb{x%6ldp&rQ#eXM#EY{JTj_6?${ zSv7k zxgtoaek#{!G@a2EqiIlWO>Y<&7X?Pks9!J54@Sl5b^5inLfu-ExkY`xtSKk8`l5D; zwMOCdH;>NI`F+}Xi+)nvx#U_ymAZVbDlYZ){aTvlTuL+gW+w}6{@?UuNf*N%5}TNV z4c3-Q)JawIPr0%-jVZ(H{076z&@V1FrFNr-zAC%8ditC?yl&?Dp{p8QtX0W#>mXJY zI%JLNH_`TmMQaaq4VwOwj-uN~BX@Q%dUkYEL8)Ho>nxwW`Qp>+S?V=o;x!z%w*8s4 zTWvF)m?wqx=ac@4*qDg)p{Sj97na4ASQ06HSy^XvYuEciSyVf7Lq(e6#l`w&atI?f zRfor|-i$98^^GrBEK-Ex`~O>UkpVt|00@8p2!H?xfB*=900@8p2!O!dC4le$-(8-t z1_B@e0w4eaAOHd&00JNY0w4eaTaf@h|KEze0-rzt1V8`;KmY_l00ck)1V8`;K;Z5Y zz~}#WmnW=&00@8p2!H?xfB*=900@8p2!Oy=B!JKVw_>lrClCMu5C8!X009sH0T2KI z5C8!XxVr@K`TyPJ32Pt#0w4eaAOHd&00JNY0w4eaAg~n)D4+kk7}3G}CwjvR1V8`; zKmY_l00ck)1V8`;KmY_l;LZq~bq$O-R{erlrzb>;b@t7VBv&3`6OUY4dh+SH`7>8u zJvAY=s`b*7%hmJE`S~j=t(jNz$t!bWCM)J9=Y<;+$!C_^^J1#}!t}MLoC8rZ#+7Rp zOO>Z@q}W`0_VSA>OHZ#%pTE9zdMZ7C{mB=vzgkK>RT3xCH`=9(h3vDJug$->(#V~C z@#(YADWCtlm_K(gU!ga=KmY_l00ck)1V8`;KmY_l00ck)1nzMHA@_(=Uwv@-+)-`y z0Kfmg$Ga8og8&GC00@8p2!H?xfB*=900@A*;Q59YnhyO_5!O{T(p6Enx0VzSI* z%s%EJW-#*ikuOI6B=SEZACLTKGik3WZ$3uLi#u z{QrW#8~pFVPXvE9_#cBG3jUkmcavLqfdB}A00@8p2!H?xfWUo9!0(Pa<&}oeI%{k_ zWo*qFTc?bzCycE(7+YCmD`jjYjIE=_)?>!jn6dSUv30=M+GlL-HnyV1)=pz9U~GAf zE%$c!n6uF?RYksdT_}~g_>haam}#M5+F?t@KG->5Q#(%2xWg zt@N0!G-E4G+Da#FrQ^2JxUKY2Tj?QNY0OqSVk;fCmF}{Y?y!~mZKZ>@QrECM=d6g$ zmR#)gvEHMho$jMlLUB`+g&PzjqAWEFJKRS)!bZ8WEDG0!W?qRLq0W^kJkD6QPg}O9 zE!!t8+b1mBIm>q1vdvnyCoJ1XEZc`I+oP84gO=_6mhHWk?T0PfLzeA_EZaWI_CUbB z$KPr1E;s8`1_=$CVvFbMMHSeds5|8>as1WArqr&NsjyY7No66Z<^PX6n7?K|%lt0$ z%goQv3cwFC-@&|%sWSreGV>hsG&9BInMr1h+0R5H{}lPt$gf3yF7jiMe;fIp$hSpS zBiAEkS_ODKayar(_#eW57XDQDKZk!T{IA3BqIH0U@Y!%aJQ{u|^!3nRgnlRViO@$v zKM?vJbtOO!iJ>=!&V`PJ4u^IG|0(!6%Ktw}+5d-w?+xAxR)d#jqAOHd& z00JNY0w8dY5>PI-wS2WM$znz1Ihiugv-aYr?Zsyva>t#rP-|2v?bULcr}{-t_KQwO z+y|Vk6?&XlP39I`v)#Kh-MiCD?$l^X%c7DSi}l4K*AP$H38(CYCzb5n+Q!xseUiuf zBqz-D{tf2ladY#Sxmhqb^X6twX||dSHrw^<;;USG3SWoneCi_JbcG>AsJK?yU@Q6ZBOKw|BT$6g}u|7$>Pjc8M`yI29J!&Hx zRnl**#u~*Pr{&I%*h_vnJEJQLW!0tzGP+j5uC)q=yzU)NTCm#L`Maa@_aUEq$jM7h zVY|MuP2XVjjflPx);B`>Mo?M*;-z{;Tx<#rsVN6~1^!-vPg(flrCLp>%Pp^UXVAJc zphQMjey^^Wal9-pi}H$F(baccLHCGX-@*6)@6l5a+y(&<009sH0T2KI5C8!X009sH zf!iQ}{Qov!K@|i*00ck)1V8`;KmY_l00ck)1nyA+$p7!rj)U7E00JNY0w4eaAOHd& z00JNY0w8c31eE;W&3wVZe2wk^@P*qr5UL;m0w4eaAOHd&00JNY0w4eaAOHeelfaK~WO-Vb|&gKr!7 zO(be z^wNlP^$D?F7H){;8%2pP@?5jrDsts=Q)snHl5|xp7u!v-sOsqD?dc1%Qx|7rXXa*R zUyL2x%%KNk^K-F-x`BhS!wX$?EQ~KuEerA0aMc@qF78}C(&bcBC<}F2NTqubx9U0BG1X9RBsi}%)D4+v#0WjTsqIP z)2FhhQd7xPawd~ao+3tF%1v9}x>VM+XSmySh~lwSx+3s$QC?{X)TNz0u}n=kD0-ts zYQpU1nlP1_$ty3L zoz(5D?PMn!_bzJw6*lMJbRj#NOH7~c^-sD@|8zTR`^PmDeEy0zdY1Zl%<3M;^HRG` zF(EW-E%n_PEnP)*Tc^5IEJquzbSWqysMJHg>=C@t7s!{c*{w^tsP9?EuP#!Zt!=_1 z{Q%v@+J2du%wK$L}A z2e@)3?sj5%9VcuL>~};}>zw1Y8r7ZnYG%P36;jSst;WuEAzdo9#Fk8>p;(pptJBMwuCX+ceT}ZIg+0)7?s#kB(*3n)q^+KiBn&EPq65q`4h|WqUQH`Yv}YI z+f`E`HGL{q$WGh%UAcX~b!+STt#-e1Ek#3da#O=mQ(B#Vqc?i3;9M20ty`uGM`2NF zt`ucSpNCADHZhskwPuN-E!Z~GMzLthy387lHOmd-o#}{LJHF6~tphE}3@F~aV#^xu z+JNg1_&3=)y55n7XR4Coi#9OS8ptC{%#UkDnvZX7t~n=I40I32S2yXM+IQAOHd&00JNY0w4eaAOHd&00JrjuiNA2ydUC_|05tk00ck)1V8`;KmY_l00ck) z1VG^aC7|X1?|T;a|GR(t7%c+<5C8!X009sH0T2KI5C8!X00Er<@_(HK{(=AqfB*=9 z00@8p2!H?xfB*=9!2L@=$^YHVA3K=8Wj;@T@B#r4009sH0T2KI5C8!X009sH0T8$c z2?X53PW8zyvfJEr`Q5{QO{(SppLQ@`VZOk8`W|c`Tm}IU009sH0T2KI5C8!X009sH z0T56K_}x(_SFVY5x3T5&xubqnrse;iq5JGp9HpfhNGRY_9rH?6Rdl;=V;Vf5vxM0C<`}ap)AT$GdYo(U|qwWBg1~Js8Oyg zi^6rGnV4Xc6Gq387UW00JNY0w4eaAOHd&00JNY0w4ea8Uf`0hy)M-0T2KI z5C8!X009sH0T2KI5V-#dDEYsO`E>{LIeNnj1V8`;KmY_l00ck)1V8`;KmY_l;2t2b z+vCqUXE>QFaV_DPAl6$ly-h6h71u70pY>b?IIxsv}onZI$+KfFKy1V8`;KmY_l z00ck)1V8`;KmY{pCjtS_u+LwSs%4?6<^NxEFkic$8i=-l00@8p2!H?xfB*=900@8p z2!H?xbO?CdPQQBpe<$+~4*G`|2!H?xfB*=900@8p2!H?xfB*=9zEt0|5{K0T2KI5C8!X009sH z0T2Lz2arH`V7qUaGJo^_e~xVrAOl7Jr_P)UD_hg&<}b~Jj4h9r|Nl37{vUk<;4kSr0H0<4C#?efCiCCE5l+J<2!H?x zfB*=900@8p2!H?xfB*=9z+E6P;C4D)Vf8(vzSEKbNdoG--{bZ>J<6L~d2_kle)ai( zLFV5$n6ENlVm`zC0e$}eN#^79D}bM1K1gD`KmY_l00ck)1V8`;KmY_l00ck)1is+} zd~T1E^SfIl8@qh^hF9Mh)Heq74Nqrd$gOXk5+_+He2O{6FfTU;qCW zeeVCGl>6Ue>da+khM8od>opK%AOHd&00JNY0w4eaAOHd&00JOz&l89aj8L+!-?Gn0 z+Jid>qDB&LWb!t`QNN9Fm(Nbs`y7GTr*p>eqLTmnS2(9_QA++ZIhR~Y->%y-5-w;~!e?u4!eBQCc z`9%lQiu`(HJp9(cqW@R@kNQ@>FU(F|oQ<8Co0)wvc5t&M4#wu^Vh1gT4#o~I7}YGqudNPvqtkm_ ztBlpND^jUdq_-kp za6=ZFb*@VDfVtgn2~BE4w}#_Aww`n=^_el8>ef@-Q)(gaoU3kc^vsxRb;Ro3VpD22 zii%2cv0ZFiy2vbXyxZ0zQ=#in!xOVy^+GI@6Z=+O-sq*2YxN1M69ycqA&XMIRn-0# zYf`%|6QJI5$G6!!)T`EI#&D}wOL0!Eh~kqxjIKJp(etCO)hDeUDl;6<@k>InLQ}ac zHMvEh*b-kA$UCFV@h)4pjPl-N4VMfJ#g~rth4BT&KK)TjRo8Z}ItU)QHoKu~HL)hp za8TeG#vSk6ynFib8@Z_KDbfG7+i^}_X>_xD$L%?%9KRXo6g_fo=&hP_$JZp+nLO%T zO(-sLt)(t<-@37Ly~o=-r)g2AD)o?G`L}qZJnLFrwE9I4^bzW1uE{TP^+mdL(D!L@ zwpQxAEL|1qs5!pH*2x|%<0z{!!`B{VB^-4TMIc3_UOl>DOnKvYkx7+$@9AIo{M;gBAa>XgFsHk@CxW1>3k%MpYMxP_EF0b#^ z|KHyI#<*=3a2&VeB#v{JJnhDo{Sj){QI%7>!RUslrcFDp1&waKmUb(MLpe9`-FcV9 z!EtVP)5@@z_orU*@|@ zQL8oWH#)`7<@w{EKTe!Hmy07G6!LgnHQViEvPVZuHt3z2&RkfKn{3s?1Mj|hxe`g| z$wy?zli$ZA>27#PuN)gWI^X1s1K%6NEl$ncpX=$FjskC-v; zJW9r7ICWL8d{6@3xOuqYT%Q^-SI{ptQX+$*WS+d3U!0NcvJ>$4KB!l${i?t2FRw&_^~tGI(kBJ>*1k#$BJJ0m`r;| z@0wBfUI*i0T~hMRoy9D)u8+B9SD*?Cu2PH$ww>8%s4n2oL#&nSi?o<4_& ziIFzV7`Bs7CPhyksHaqaPcmfm+B^IULp52JFYu*c;+BU~lO-A?8wc})I8&lcuv=x- z{8aS$|I>>2xA==Z``<6+834X5KLzk*@svCV!0F5G0m%jVBO-tR0tg_000IagfB*srAb`NdBM{90%VL)zUKjtA-~ab3@gwma@v``WJPY88 zsEfnmeX^1Z0tg_000IagfB*srAbW5b4N68x2%}(7bOlH+< z)qMED-+KAlth!&#I}f&I8cw&q^Io&nl_R$63#F*p(Iv-oI)=!qJJkF^r`8QWt1bTl zK&`0R=w1A%9Ddw2cV*U?ivFn}{uHh@nKiCg%@xx#-A*Q-RZHp{*_>P$Z$$I|7ZmXy z`3-=th(F5D0Q^?`Li|L0U;Z878{%u?tKx-AeiyTW2q1s}0tg_000IagfB*sryoCaK zR#nwZM(ZtuiMwiKt@_(pyNtsu4f{0R#|0009ILKmY**5V(8<bjmY@#6_Grh$UiX^r zvgOUqELpYXXoPj`M35*L{R4Kx>QwKy+>Q*?nOoOr?$B!6cCCJ8`Q4h8=ySp~J4>hR zR%)NHvUID~OmsNDWV&{1QI6_aZp&=U9hcvA*Z(D`v^sJNCUqbqt%RsY009ILKmY**5I_I{1Q0-A`~{f* zkN-Yo3lKm60R#|0009ILKmY**5ZFq9`Ttfz)FXfZ0tg_000IagfB*srATa&{!TdiX z{wlx!PhN3B009ILKmY**5I_I{1Q0*~fiV}D)ZVUIc1xb_&~9bceg9uZ{6P_C5I_I{ z1Q0*~0R#|0U~~cI|D#*c5&;AdKmY**5I_I{1Q0*~fpHZG=KrdARi6KUT(^MS2q1s} z0tg_000IagfB*srAh3bJy7T|5;;cOX{{}6&836uOffAAb2zD(2Jyf6|HKam+$wq4sA|_`>M8$9y<+dq__N8>X4hNt zoMo$39h4eo$E%t(&t9>;wfGJZ*DEii#*OPYA1O{8w-1EZt#W?q5&sIka(G9^-;vB3 zZcL}zTeyzn#o?mcl+ULo+o)e0C2A1lhz1R~3jTJza!f|pyK#h$=eXvgHJsx45mKz{ zni3?gmsuY1w{5A>?O5(`j^{?mG3b{PBB~5D^1hG}F2{_yit5BX)iG9!aj(u0JDiZ_RNW{VJmZs`Gn`uV8`{gPgp zmMgz9edS%d9`A!Vzw-1J>yCS+uUb~i9dMmLsaKBQlJRd(t~x$m)tuIXy;$un!uxab zSQFi@Je8Vcvq9oWLydx+60RjYwygORa;Mxlv{TGlEjZ?Ns#epq2Tv^ho0ZR{rs-EF zd&MdGmBH%6QKd}TFUkk1T>1W_OuuEdZg<)ZbFJDkn}Y{xzgBs2^9*n9oEjzBOeVVC zH}uMVGSS`1L^Y?`v;vN}AP=$5;6dBJS@~>grhav5_c&F*RR9zXAKr8K^?nKOO1Jt2 zy>e2~umZ literal 0 HcmV?d00001 From e9b70c5efe970019af8c186c7f5fa51349ed8121 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 19:25:18 +0200 Subject: [PATCH 18/25] go fmt --- cmd/revad/runtime/runtime.go | 7 ------- internal/grpc/interceptors/auth/auth.go | 2 +- internal/grpc/interceptors/auth/scope.go | 2 +- internal/grpc/interceptors/metrics/metrics.go | 2 +- internal/grpc/interceptors/trace/trace.go | 1 - internal/grpc/services/pingpong/pingpong.go | 4 ++-- internal/http/interceptors/auth/auth.go | 2 +- .../http/interceptors/auth/token/strategy/header/header.go | 2 +- .../auth/tokenwriter/strategy/header/header.go | 2 +- internal/http/interceptors/metrics/metrics.go | 5 +++-- internal/http/services/pingpong/pingpong.go | 6 ++---- pkg/auth/manager/nextcloud/nextcloud_test.go | 2 +- pkg/httpclient/httpclient.go | 7 +------ pkg/ocm/share/repository/nextcloud/nextcloud_test.go | 2 +- pkg/prom/base/base.go | 3 ++- pkg/prom/loader/loader.go | 2 +- pkg/prom/registry/registry.go | 3 ++- pkg/share/cache/warmup/cbox/cbox.go | 2 +- pkg/storage/favorite/sql/sql.go | 2 +- pkg/storage/fs/local/local.go | 4 ++-- pkg/storage/fs/localhome/localhome.go | 6 +++--- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 +- pkg/user/manager/nextcloud/nextcloud_test.go | 2 +- tests/integration/grpc/ocm_invitation_test.go | 2 +- 24 files changed, 31 insertions(+), 43 deletions(-) diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 44fa47a530..6dbdfcc4a6 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -85,7 +85,6 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { if err := initCPUCount(config.Core, log); err != nil { return nil, err } - initTracing(config.Core) if opts.PidFile == "" { return nil, errors.New("pid file not provided") @@ -346,12 +345,6 @@ func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { return w, nil } -func initTracing(conf *config.Core) { - if conf.TracingEnabled { - //TODO(labkode): init this - } -} - // adjustCPU parses string cpu and sets GOMAXPROCS // according to its value. It accepts either // a number (e.g. 3) or a percent (e.g. 50%). diff --git a/internal/grpc/interceptors/auth/auth.go b/internal/grpc/interceptors/auth/auth.go index ee6b384dd3..511a921972 100644 --- a/internal/grpc/interceptors/auth/auth.go +++ b/internal/grpc/interceptors/auth/auth.go @@ -27,8 +27,8 @@ import ( gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index b1c82122ac..b3e01fe969 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -36,8 +36,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" statuspkg "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/grpc/interceptors/metrics/metrics.go b/internal/grpc/interceptors/metrics/metrics.go index 899112ef64..5c18144122 100644 --- a/internal/grpc/interceptors/metrics/metrics.go +++ b/internal/grpc/interceptors/metrics/metrics.go @@ -33,7 +33,7 @@ func init() { registry.Register("grpc_metrics", NewPromCollectors) } -// New returns a prometheus collector +// New returns a prometheus collector. func NewPromCollectors(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { return []prometheus.Collector{collector}, nil } diff --git a/internal/grpc/interceptors/trace/trace.go b/internal/grpc/interceptors/trace/trace.go index 345497a0bd..aafffd9b28 100644 --- a/internal/grpc/interceptors/trace/trace.go +++ b/internal/grpc/interceptors/trace/trace.go @@ -30,7 +30,6 @@ func getContext(ctx context.Context) context.Context { traceID := trace.Get(ctx) if traceID != "" { return ctx - } md, ok := metadata.FromIncomingContext(ctx) diff --git a/internal/grpc/services/pingpong/pingpong.go b/internal/grpc/services/pingpong/pingpong.go index f8f94ea7f7..b36b1bd490 100644 --- a/internal/grpc/services/pingpong/pingpong.go +++ b/internal/grpc/services/pingpong/pingpong.go @@ -82,7 +82,7 @@ func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingRe } client := httpclient.New(httpclient.RoundTripper(tr)) - req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/ping", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.conf.Endpoint+"/ping", nil) if err != nil { log.Error().Err(err).Msg("error creating http request") return nil, err @@ -113,7 +113,7 @@ func (s *service) Pong(ctx context.Context, _ *proto.PongRequest) (*proto.PongRe } client := httpclient.New(httpclient.RoundTripper(tr)) - req, err := http.NewRequestWithContext(ctx, "GET", s.conf.Endpoint+"/pong", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.conf.Endpoint+"/pong", nil) if err != nil { log.Error().Err(err).Msg("error creating http request") return nil, err diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index ee8470ce0f..094e81ac8a 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -33,9 +33,9 @@ import ( tokenregistry "github.com/cs3org/reva/internal/http/interceptors/auth/token/registry" tokenwriterregistry "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/registry" "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" diff --git a/internal/http/interceptors/auth/token/strategy/header/header.go b/internal/http/interceptors/auth/token/strategy/header/header.go index 558ea3eba3..3f1c30bcf7 100644 --- a/internal/http/interceptors/auth/token/strategy/header/header.go +++ b/internal/http/interceptors/auth/token/strategy/header/header.go @@ -22,8 +22,8 @@ import ( "net/http" "github.com/cs3org/reva/internal/http/interceptors/auth/token/registry" - "github.com/cs3org/reva/pkg/auth" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth" ) func init() { diff --git a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go index 7a945eae2e..d3e689ec53 100644 --- a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go +++ b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go @@ -22,8 +22,8 @@ import ( "net/http" "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/registry" - "github.com/cs3org/reva/pkg/auth" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth" ) func init() { diff --git a/internal/http/interceptors/metrics/metrics.go b/internal/http/interceptors/metrics/metrics.go index 1311ccacb9..023292d762 100644 --- a/internal/http/interceptors/metrics/metrics.go +++ b/internal/http/interceptors/metrics/metrics.go @@ -23,10 +23,11 @@ package metrics import ( "context" + "net/http" + "github.com/cs3org/reva/pkg/prom/registry" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" ) var inFlightGauge = prometheus.NewGauge(prometheus.GaugeOpts{ @@ -79,7 +80,7 @@ func init() { registry.Register("http_metrics", NewPromCollectors) } -// New returns a prometheus collector +// New returns a prometheus collector. func NewPromCollectors(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { return []prometheus.Collector{inFlightGauge, counter, duration, responseSize}, nil } diff --git a/internal/http/services/pingpong/pingpong.go b/internal/http/services/pingpong/pingpong.go index 7dbca3fd45..31739dd01f 100644 --- a/internal/http/services/pingpong/pingpong.go +++ b/internal/http/services/pingpong/pingpong.go @@ -110,7 +110,7 @@ func (s *svc) getClient() (proto.PingPongServiceClient, error) { return client, nil } -// doPing will call the grpc Pong method +// doPing will call the grpc Pong method. func (s *svc) doPing(w http.ResponseWriter, r *http.Request) { log := appctx.GetLogger(r.Context()) client, err := s.getClient() @@ -130,11 +130,9 @@ func (s *svc) doPing(w http.ResponseWriter, r *http.Request) { log.Info().Msg("pinging from http to grpc") w.Write([]byte(pingRes.Info)) - return } -// doPong will be (http) called from grpc Pong +// doPong will be (http) called from grpc Pong. func (s *svc) doPong(w http.ResponseWriter, r *http.Request) { w.Write([]byte("pong")) - return } diff --git a/pkg/auth/manager/nextcloud/nextcloud_test.go b/pkg/auth/manager/nextcloud/nextcloud_test.go index 81bf6a82cb..26dfe9243c 100644 --- a/pkg/auth/manager/nextcloud/nextcloud_test.go +++ b/pkg/auth/manager/nextcloud/nextcloud_test.go @@ -26,9 +26,9 @@ import ( authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/manager/nextcloud" "github.com/cs3org/reva/pkg/auth/scope" - ctxpkg "github.com/cs3org/reva/pkg/appctx" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index d4ee86d6b6..20187a7954 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -1,15 +1,12 @@ package httpclient import ( - //"io" - "errors" "net/http" "time" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/trace" - //"net/url" ) // TODO(labkode): harden it. @@ -73,7 +70,6 @@ func CheckRedirect(cr func(req *http.Request, via []*http.Request) error) Option return func(o *Options) { o.CheckRedirect = cr } - } // Jar provides a function to set a custom CookieJar. @@ -81,11 +77,10 @@ func Jar(j http.CookieJar) Option { return func(o *Options) { o.Jar = j } - } // Client wraps a http.Client but only exposes the Do method -// to force consumers to always create a request with http.NewRequestWithContext() +// to force consumers to always create a request with http.NewRequestWithContext(). type Client struct { c *http.Client } diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go index b2e5375df4..7b01d9c06b 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go @@ -27,8 +27,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" masked_share "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/nextcloud" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" diff --git a/pkg/prom/base/base.go b/pkg/prom/base/base.go index 6f598d03ac..eaae139c8d 100644 --- a/pkg/prom/base/base.go +++ b/pkg/prom/base/base.go @@ -20,6 +20,7 @@ package base import ( "context" + "github.com/cs3org/reva/pkg/prom/registry" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -29,7 +30,7 @@ func init() { registry.Register("base", New) } -// New returns a prometheus collector +// New returns a prometheus collector. func New(_ context.Context, m map[string]interface{}) ([]prometheus.Collector, error) { return []prometheus.Collector{ collectors.NewBuildInfoCollector(), diff --git a/pkg/prom/loader/loader.go b/pkg/prom/loader/loader.go index f62329d8ee..d9406544a5 100644 --- a/pkg/prom/loader/loader.go +++ b/pkg/prom/loader/loader.go @@ -19,7 +19,7 @@ package loader import ( - // Load collectors + // Load collectors. _ "github.com/cs3org/reva/internal/grpc/interceptors/metrics" _ "github.com/cs3org/reva/internal/http/interceptors/metrics" _ "github.com/cs3org/reva/pkg/prom/base" diff --git a/pkg/prom/registry/registry.go b/pkg/prom/registry/registry.go index c2f60c6555..f5b9be6898 100644 --- a/pkg/prom/registry/registry.go +++ b/pkg/prom/registry/registry.go @@ -2,6 +2,7 @@ package registry import ( "context" + "github.com/prometheus/client_golang/prometheus" ) @@ -9,7 +10,7 @@ import ( // should register at init time. type NewFunc func(context.Context, map[string]interface{}) ([]prometheus.Collector, error) -// NewFuncs is a map containing all the registered collectors +// NewFuncs is a map containing all the registered collectors. var NewFuncs = map[string]NewFunc{} // Register registers a new prometheus collector new function. diff --git a/pkg/share/cache/warmup/cbox/cbox.go b/pkg/share/cache/warmup/cbox/cbox.go index cc8b55e2ef..10e031693d 100644 --- a/pkg/share/cache/warmup/cbox/cbox.go +++ b/pkg/share/cache/warmup/cbox/cbox.go @@ -26,8 +26,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share/cache" "github.com/cs3org/reva/pkg/share/cache/warmup/registry" diff --git a/pkg/storage/favorite/sql/sql.go b/pkg/storage/favorite/sql/sql.go index 8e74a8de30..46f8ed98e6 100644 --- a/pkg/storage/favorite/sql/sql.go +++ b/pkg/storage/favorite/sql/sql.go @@ -25,8 +25,8 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/cbox/utils" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/storage/favorite" "github.com/cs3org/reva/pkg/storage/favorite/registry" "github.com/cs3org/reva/pkg/utils/cfg" diff --git a/pkg/storage/fs/local/local.go b/pkg/storage/fs/local/local.go index 7ecbd04d1d..504cb835a9 100644 --- a/pkg/storage/fs/local/local.go +++ b/pkg/storage/fs/local/local.go @@ -32,8 +32,8 @@ func init() { } type config struct { - Root string `mapstructure:"root" docs:"/var/tmp/reva/;Path of root directory for user storage."` - ShareFolder string `mapstructure:"share_folder" docs:"/MyShares;Path for storing share references."` + Root string `docs:"/var/tmp/reva/;Path of root directory for user storage." mapstructure:"root"` + ShareFolder string `docs:"/MyShares;Path for storing share references." mapstructure:"share_folder"` } func (c *config) ApplyDefaults() { diff --git a/pkg/storage/fs/localhome/localhome.go b/pkg/storage/fs/localhome/localhome.go index f57534c2df..fae49c711e 100644 --- a/pkg/storage/fs/localhome/localhome.go +++ b/pkg/storage/fs/localhome/localhome.go @@ -33,9 +33,9 @@ func init() { } type config struct { - Root string `mapstructure:"root" docs:"/var/tmp/reva/;Path of root directory for user storage."` - ShareFolder string `mapstructure:"share_folder" docs:"/MyShares;Path for storing share references."` - UserLayout string `mapstructure:"user_layout" docs:"{{.Username}};Template for user home directories"` + Root string `docs:"/var/tmp/reva/;Path of root directory for user storage." mapstructure:"root"` + ShareFolder string `docs:"/MyShares;Path for storing share references." mapstructure:"share_folder"` + UserLayout string `docs:"{{.Username}};Template for user home directories" mapstructure:"user_layout"` } func (c *config) ApplyDefaults() { diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index b9053830a6..8be30a838e 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -30,8 +30,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/storage/fs/nextcloud" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" . "github.com/onsi/ginkgo" diff --git a/pkg/user/manager/nextcloud/nextcloud_test.go b/pkg/user/manager/nextcloud/nextcloud_test.go index a33821d164..88fa5ec3a6 100644 --- a/pkg/user/manager/nextcloud/nextcloud_test.go +++ b/pkg/user/manager/nextcloud/nextcloud_test.go @@ -23,8 +23,8 @@ import ( "os" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/pkg/user/manager/nextcloud" "github.com/cs3org/reva/tests/helpers" diff --git a/tests/integration/grpc/ocm_invitation_test.go b/tests/integration/grpc/ocm_invitation_test.go index 1e3626d602..6228dd82d5 100644 --- a/tests/integration/grpc/ocm_invitation_test.go +++ b/tests/integration/grpc/ocm_invitation_test.go @@ -32,8 +32,8 @@ import ( ocmproviderpb "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" From a5c056e845b451cb15c44efc05a72e7148c18151 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 19:46:50 +0200 Subject: [PATCH 19/25] more linters --- internal/grpc/interceptors/auth/auth.go | 25 +++++++++---------- internal/grpc/interceptors/auth/scope.go | 3 +-- internal/grpc/interceptors/log/log.go | 5 ++-- internal/grpc/services/gateway/appprovider.go | 7 +++--- .../grpc/services/gateway/authprovider.go | 13 +++++----- .../ocminvitemanager/ocminvitemanager.go | 13 +++++----- internal/grpc/services/pingpong/pingpong.go | 7 +++--- internal/http/interceptors/auth/auth.go | 4 +-- internal/http/interceptors/trace/trace.go | 5 +--- .../ocs/handlers/cloud/users/users.go | 3 +-- pkg/app/provider/wopi/wopi.go | 7 +++--- pkg/appctx/pathctx.go | 18 +++++++++++++ pkg/httpclient/httpclient.go | 17 +++++++++++++ pkg/trace/trace.go | 3 ++- pkg/user/manager/nextcloud/nextcloud.go | 3 +-- 15 files changed, 77 insertions(+), 56 deletions(-) diff --git a/internal/grpc/interceptors/auth/auth.go b/internal/grpc/interceptors/auth/auth.go index 511a921972..ed42c2939e 100644 --- a/internal/grpc/interceptors/auth/auth.go +++ b/internal/grpc/interceptors/auth/auth.go @@ -27,7 +27,6 @@ import ( gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -102,21 +101,21 @@ func NewUnary(m map[string]interface{}, unprotected []string) (grpc.UnaryServerI // If a token is present, set it anyway, as we might need the user info // to decide the storage provider. - tkn, ok := ctxpkg.ContextGetToken(ctx) + tkn, ok := appctx.ContextGetToken(ctx) if ok { u, scopes, err := dismantleToken(ctx, tkn, req, tokenManager, conf.GatewayAddr, true) if err == nil { if blockedUsers.IsBlocked(u.Username) { return nil, status.Errorf(codes.PermissionDenied, "user %s blocked", u.Username) } - ctx = ctxpkg.ContextSetUser(ctx, u) - ctx = ctxpkg.ContextSetScopes(ctx, scopes) + ctx = appctx.ContextSetUser(ctx, u) + ctx = appctx.ContextSetScopes(ctx, scopes) } } return handler(ctx, req) } - tkn, ok := ctxpkg.ContextGetToken(ctx) + tkn, ok := appctx.ContextGetToken(ctx) if !ok || tkn == "" { log.Warn().Msg("access token not found or empty") @@ -134,8 +133,8 @@ func NewUnary(m map[string]interface{}, unprotected []string) (grpc.UnaryServerI return nil, status.Errorf(codes.PermissionDenied, "user %s blocked", u.Username) } - ctx = ctxpkg.ContextSetUser(ctx, u) - ctx = ctxpkg.ContextSetScopes(ctx, scopes) + ctx = appctx.ContextSetUser(ctx, u) + ctx = appctx.ContextSetScopes(ctx, scopes) return handler(ctx, req) } return interceptor, nil @@ -175,12 +174,12 @@ func NewStream(m map[string]interface{}, unprotected []string) (grpc.StreamServe // If a token is present, set it anyway, as we might need the user info // to decide the storage provider. - tkn, ok := ctxpkg.ContextGetToken(ctx) + tkn, ok := appctx.ContextGetToken(ctx) if ok { u, scopes, err := dismantleToken(ctx, tkn, ss, tokenManager, conf.GatewayAddr, true) if err == nil { - ctx = ctxpkg.ContextSetUser(ctx, u) - ctx = ctxpkg.ContextSetScopes(ctx, scopes) + ctx = appctx.ContextSetUser(ctx, u) + ctx = appctx.ContextSetScopes(ctx, scopes) ss = newWrappedServerStream(ctx, ss) } } @@ -188,7 +187,7 @@ func NewStream(m map[string]interface{}, unprotected []string) (grpc.StreamServe return handler(srv, ss) } - tkn, ok := ctxpkg.ContextGetToken(ctx) + tkn, ok := appctx.ContextGetToken(ctx) if !ok || tkn == "" { log.Warn().Msg("access token not found") @@ -203,8 +202,8 @@ func NewStream(m map[string]interface{}, unprotected []string) (grpc.StreamServe } // store user and core access token in context. - ctx = ctxpkg.ContextSetUser(ctx, u) - ctx = ctxpkg.ContextSetScopes(ctx, scopes) + ctx = appctx.ContextSetUser(ctx, u) + ctx = appctx.ContextSetScopes(ctx, scopes) wrapped := newWrappedServerStream(ctx, ss) return handler(srv, wrapped) } diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index b3e01fe969..c3cd7bec3e 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -36,7 +36,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" statuspkg "github.com/cs3org/reva/pkg/rgrpc/status" @@ -305,7 +304,7 @@ func checkIfNestedResource(ctx context.Context, ref *provider.Reference, parent if err != nil { return false, err } - ctx = metadata.AppendToOutgoingContext(context.Background(), ctxpkg.TokenHeader, token) + ctx = metadata.AppendToOutgoingContext(context.Background(), appctx.TokenHeader, token) childStat, err := client.Stat(ctx, &provider.StatRequest{Ref: ref}) if err != nil { diff --git a/internal/grpc/interceptors/log/log.go b/internal/grpc/interceptors/log/log.go index 3c617da422..99805a4a97 100644 --- a/internal/grpc/interceptors/log/log.go +++ b/internal/grpc/interceptors/log/log.go @@ -23,7 +23,6 @@ import ( "time" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/rs/zerolog" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -44,7 +43,7 @@ func NewUnary() grpc.UnaryServerInterceptor { if p, ok := peer.FromContext(ctx); ok { fromAddress = p.Addr.Network() + "://" + p.Addr.String() } - userAgent, ok := ctxpkg.ContextGetUserAgentString(ctx) + userAgent, ok := appctx.ContextGetUserAgentString(ctx) if !ok { userAgent = "" } @@ -86,7 +85,7 @@ func NewStream() grpc.StreamServerInterceptor { if p, ok := peer.FromContext(ss.Context()); ok { fromAddress = p.Addr.Network() + "://" + p.Addr.String() } - userAgent, ok := ctxpkg.ContextGetUserAgentString(ctx) + userAgent, ok := appctx.ContextGetUserAgentString(ctx) if !ok { userAgent = "" } diff --git a/internal/grpc/services/gateway/appprovider.go b/internal/grpc/services/gateway/appprovider.go index 28f17abd00..7a317df61b 100644 --- a/internal/grpc/services/gateway/appprovider.go +++ b/internal/grpc/services/gateway/appprovider.go @@ -33,7 +33,6 @@ import ( storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -158,8 +157,8 @@ func (s *svc) openFederatedShares(ctx context.Context, targetURL string, req *ga } gatewayClient := gateway.NewGatewayAPIClient(conn) - remoteCtx := ctxpkg.ContextSetToken(context.Background(), ep.token) - remoteCtx = metadata.AppendToOutgoingContext(remoteCtx, ctxpkg.TokenHeader, ep.token) + remoteCtx := appctx.ContextSetToken(context.Background(), ep.token) + remoteCtx = metadata.AppendToOutgoingContext(remoteCtx, appctx.TokenHeader, ep.token) res, err := gatewayClient.OpenInApp(remoteCtx, appProviderReq) if err != nil { @@ -170,7 +169,7 @@ func (s *svc) openFederatedShares(ctx context.Context, targetURL string, req *ga } func (s *svc) openLocalResources(ctx context.Context, ri *storageprovider.ResourceInfo, req *gateway.OpenInAppRequest) (*providerpb.OpenInAppResponse, error) { - accessToken, ok := ctxpkg.ContextGetToken(ctx) + accessToken, ok := appctx.ContextGetToken(ctx) if !ok || accessToken == "" { return &providerpb.OpenInAppResponse{ Status: status.NewUnauthenticated(ctx, errtypes.InvalidCredentials("Access token is invalid or empty"), ""), diff --git a/internal/grpc/services/gateway/authprovider.go b/internal/grpc/services/gateway/authprovider.go index d0a405de62..da485f93c6 100644 --- a/internal/grpc/services/gateway/authprovider.go +++ b/internal/grpc/services/gateway/authprovider.go @@ -29,7 +29,6 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -112,9 +111,9 @@ func (s *svc) Authenticate(ctx context.Context, req *gateway.AuthenticateRequest return res, nil } - ctx = ctxpkg.ContextSetToken(ctx, token) - ctx = ctxpkg.ContextSetUser(ctx, res.User) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, token) + ctx = appctx.ContextSetToken(ctx, token) + ctx = appctx.ContextSetUser(ctx, res.User) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) // Commenting out as the token size can get too big // For now, we'll try to resolve all resources on every request and cache those @@ -148,9 +147,9 @@ func (s *svc) Authenticate(ctx context.Context, req *gateway.AuthenticateRequest // we need to pass the token to authenticate the CreateHome request. // TODO(labkode): appending to existing context will not pass the token. - ctx = ctxpkg.ContextSetToken(ctx, token) - ctx = ctxpkg.ContextSetUser(ctx, res.User) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, token) // TODO(jfd): hardcoded metadata key. use PerRPCCredentials? + ctx = appctx.ContextSetToken(ctx, token) + ctx = appctx.ContextSetUser(ctx, res.User) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) // TODO(jfd): hardcoded metadata key. use PerRPCCredentials? // create home directory if _, err = s.createHomeCache.Get(res.User.Id.OpaqueId); err != nil { diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index 30df2d5328..bcd0c4c919 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -27,7 +27,6 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/invite" @@ -130,7 +129,7 @@ func (s *service) UnprotectedEndpoints() []string { } func (s *service) GenerateInviteToken(ctx context.Context, req *invitepb.GenerateInviteTokenRequest) (*invitepb.GenerateInviteTokenResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) token := CreateToken(s.conf.tokenExpiration, user.GetId(), req.Description) if err := s.repo.AddToken(ctx, token); err != nil { @@ -146,7 +145,7 @@ func (s *service) GenerateInviteToken(ctx context.Context, req *invitepb.Generat } func (s *service) ListInviteTokens(ctx context.Context, req *invitepb.ListInviteTokensRequest) (*invitepb.ListInviteTokensResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) tokens, err := s.repo.ListTokens(ctx, user.Id) if err != nil { return &invitepb.ListInviteTokensResponse{ @@ -160,7 +159,7 @@ func (s *service) ListInviteTokens(ctx context.Context, req *invitepb.ListInvite } func (s *service) ForwardInvite(ctx context.Context, req *invitepb.ForwardInviteRequest) (*invitepb.ForwardInviteResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) ocmEndpoint, err := getOCMEndpoint(req.GetOriginSystemProvider()) if err != nil { @@ -329,7 +328,7 @@ func (s *service) GetAcceptedUser(ctx context.Context, req *invitepb.GetAccepted } func getUserFilter(ctx context.Context, req *invitepb.GetAcceptedUserRequest) (*userpb.User, bool) { - user, ok := ctxpkg.ContextGetUser(ctx) + user, ok := appctx.ContextGetUser(ctx) if ok { return user, true } @@ -351,7 +350,7 @@ func getUserFilter(ctx context.Context, req *invitepb.GetAcceptedUserRequest) (* } func (s *service) FindAcceptedUsers(ctx context.Context, req *invitepb.FindAcceptedUsersRequest) (*invitepb.FindAcceptedUsersResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) acceptedUsers, err := s.repo.FindRemoteUsers(ctx, user.GetId(), req.GetFilter()) if err != nil { return &invitepb.FindAcceptedUsersResponse{ @@ -366,7 +365,7 @@ func (s *service) FindAcceptedUsers(ctx context.Context, req *invitepb.FindAccep } func (s *service) DeleteAcceptedUser(ctx context.Context, req *invitepb.DeleteAcceptedUserRequest) (*invitepb.DeleteAcceptedUserResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if err := s.repo.DeleteRemoteUser(ctx, user.Id, req.RemoteUserId); err != nil { return &invitepb.DeleteAcceptedUserResponse{ Status: status.NewInternal(ctx, err, "error deleting remote users: "+err.Error()), diff --git a/internal/grpc/services/pingpong/pingpong.go b/internal/grpc/services/pingpong/pingpong.go index b36b1bd490..1056c78113 100644 --- a/internal/grpc/services/pingpong/pingpong.go +++ b/internal/grpc/services/pingpong/pingpong.go @@ -13,7 +13,6 @@ // limitations under the License. // // In applying this license, CERN does not waive the privileges and immunities - // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. @@ -22,7 +21,7 @@ package pingpong import ( "context" "crypto/tls" - "io/ioutil" + "io" "net/http" "github.com/cs3org/reva/internal/grpc/services/pingpong/proto" @@ -95,7 +94,7 @@ func (s *service) Ping(ctx context.Context, _ *proto.PingRequest) (*proto.PingRe } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { log.Error().Err(err).Msg("error reading response body") return nil, err @@ -126,7 +125,7 @@ func (s *service) Pong(ctx context.Context, _ *proto.PongRequest) (*proto.PongRe } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { log.Error().Err(err).Msg("error reading response body") return nil, err diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index 094e81ac8a..75154e3b7b 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -167,15 +167,13 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err } log := appctx.GetLogger(r.Context()) - isUnprotectedEndpoint := false // For unprotected URLs, we try to authenticate the request in case some service needs it, // but don't return any errors if it fails. if utils.Skip(r.URL.Path, unprotected) { log.Info().Msg("skipping auth check for: " + r.URL.Path) - isUnprotectedEndpoint = true } else { - ctx, err := authenticateUser(w, r, conf, tokenStrategyChain, tokenManager, tokenWriter, credChain, isUnprotectedEndpoint) + ctx, err := authenticateUser(w, r, conf, tokenStrategyChain, tokenManager, tokenWriter, credChain, false) if err != nil { return } diff --git a/internal/http/interceptors/trace/trace.go b/internal/http/interceptors/trace/trace.go index f4bec73216..cdb5906fcd 100644 --- a/internal/http/interceptors/trace/trace.go +++ b/internal/http/interceptors/trace/trace.go @@ -31,10 +31,7 @@ import ( // New returns a new HTTP middleware that stores the log // in the context with request ID information. func New() func(http.Handler) http.Handler { - chain := func(h http.Handler) http.Handler { - return handler(h) - } - return chain + return handler } func handler(h http.Handler) http.Handler { diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go index 4e440a2abd..b644ce2309 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go @@ -31,7 +31,6 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/go-chi/chi/v5" ) @@ -86,7 +85,7 @@ func (h *Handler) GetUsers(w http.ResponseWriter, r *http.Request) { user := chi.URLParam(r, "userid") // FIXME use ldap to fetch user info - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "missing user in context", fmt.Errorf("missing user in context")) return diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index acf59afa64..6d146383e2 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -44,7 +44,6 @@ import ( "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/httpclient" @@ -161,7 +160,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc q.Add("appname", p.conf.AppName) var ut = invalid - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { // we must have been authenticated return nil, errors.New("wopi: ContextGetUser failed") @@ -173,7 +172,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp) } - scopes, ok := ctxpkg.ContextGetScopes(ctx) + scopes, ok := appctx.ContextGetScopes(ctx) if !ok { // we must find at least one scope (as owner or sharee) return nil, errors.New("wopi: ContextGetScopes failed") @@ -442,7 +441,7 @@ func getAppURLs(c *config) (map[string]map[string]string, error) { } func (p *wopiProvider) getAccessTokenTTL(ctx context.Context) (string, error) { - tkn := ctxpkg.ContextMustGetToken(ctx) + tkn := appctx.ContextMustGetToken(ctx) token, err := jwt.ParseWithClaims(tkn, &jwt.StandardClaims{}, func(token *jwt.Token) (interface{}, error) { return []byte(p.conf.JWTSecret), nil }) diff --git a/pkg/appctx/pathctx.go b/pkg/appctx/pathctx.go index bec9996ed0..4bcf1e7354 100644 --- a/pkg/appctx/pathctx.go +++ b/pkg/appctx/pathctx.go @@ -1,3 +1,21 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package appctx import "context" diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index 20187a7954..5da392a402 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -1,3 +1,20 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. package httpclient import ( diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index 233a9c35c5..aff75a4c3d 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ // In applying this license, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. + package trace import ( diff --git a/pkg/user/manager/nextcloud/nextcloud.go b/pkg/user/manager/nextcloud/nextcloud.go index 46010c0664..0fbf221289 100644 --- a/pkg/user/manager/nextcloud/nextcloud.go +++ b/pkg/user/manager/nextcloud/nextcloud.go @@ -29,7 +29,6 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" @@ -104,7 +103,7 @@ func (um *Manager) SetHTTPClient(c *http.Client) { } func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { err := errors.Wrap(errtypes.UserRequired(""), "nextcloud user manager: error getting user from ctx") return nil, err From 197d54671d9bc125fd28d83d61da33cf7c7e3cdf Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 20:16:47 +0200 Subject: [PATCH 20/25] make all linters happy --- cmd/reva/download.go | 5 ++-- cmd/reva/grpc.go | 7 +++--- cmd/reva/upload.go | 5 ++-- internal/grpc/interceptors/token/token.go | 14 +++++------ .../grpc/interceptors/useragent/useragent.go | 10 ++++---- internal/grpc/services/datatx/datatx.go | 6 ++--- .../grpc/services/gateway/ocmshareprovider.go | 4 ++-- .../grpc/services/gateway/storageprovider.go | 4 ++-- .../services/gateway/usershareprovider.go | 6 ++--- .../services/gateway/webdavstorageprovider.go | 13 ++++++----- .../ocmshareprovider/ocmshareprovider.go | 19 +++++++-------- .../services/pingpong/proto/pingpong.pb.go | 5 ++-- .../pingpong/proto/pingpong_grpc.pb.go | 1 + internal/http/interceptors/auth/auth.go | 12 +++++----- .../auth/token/strategy/header/header.go | 5 ++-- .../tokenwriter/strategy/header/header.go | 5 ++-- internal/http/services/overleaf/overleaf.go | 6 ++--- internal/http/services/owncloud/ocdav/dav.go | 18 +++++++-------- .../http/services/owncloud/ocdav/ocdav.go | 4 ++-- .../http/services/owncloud/ocdav/propfind.go | 16 ++++++------- .../http/services/owncloud/ocdav/proppatch.go | 6 ++--- .../http/services/owncloud/ocdav/report.go | 3 +-- .../http/services/owncloud/ocdav/trashbin.go | 4 ++-- internal/http/services/owncloud/ocs/cache.go | 12 +++++----- .../ocs/handlers/apps/sharing/shares/group.go | 5 ++-- .../handlers/apps/sharing/shares/public.go | 6 ++--- .../handlers/apps/sharing/shares/shares.go | 4 ++-- .../ocs/handlers/apps/sharing/shares/user.go | 6 ++--- .../handlers/cloud/capabilities/uploads.go | 5 ++-- .../owncloud/ocs/handlers/cloud/user/user.go | 5 ++-- internal/http/services/pingpong/pingpong.go | 8 ++++--- .../services/reverseproxy/reverseproxy.go | 6 ++--- internal/http/services/sciencemesh/token.go | 4 ++-- pkg/appauth/manager/json/json.go | 9 ++++---- pkg/appauth/manager/json/json_test.go | 11 +++++---- pkg/auth/manager/nextcloud/nextcloud_test.go | 9 ++++---- pkg/cbox/share/sql/sql.go | 23 ++++++++++--------- .../storage/eoshomewrapper/eoshomewrapper.go | 5 ++-- pkg/cbox/storage/eoswrapper/eoswrapper.go | 7 +++--- pkg/eosclient/eosbinary/eosbinary.go | 6 ++--- pkg/httpclient/httpclient.go | 3 ++- .../repository/nextcloud/nextcloud_test.go | 9 ++++---- pkg/ocm/storage/outcoming/ocm.go | 9 ++++---- pkg/preferences/memory/memory.go | 6 ++--- pkg/preferences/sql/sql.go | 6 ++--- pkg/prom/registry/registry.go | 18 +++++++++++++++ pkg/sdk/common/net/net.go | 4 ++-- pkg/share/cache/warmup/cbox/cbox.go | 5 ++-- pkg/share/manager/json/json.go | 21 +++++++++-------- pkg/share/manager/memory/memory.go | 21 +++++++++-------- pkg/storage/favorite/memory/memory_test.go | 8 +++---- pkg/storage/favorite/sql/sql.go | 9 ++++---- pkg/storage/fs/cback/cback.go | 9 ++++---- pkg/storage/fs/nextcloud/nextcloud.go | 4 ++-- pkg/storage/fs/nextcloud/nextcloud_test.go | 9 ++++---- pkg/storage/registry/static/static.go | 5 ++-- pkg/storage/registry/static/static_test.go | 7 +++--- pkg/storage/utils/eosfs/eosfs.go | 12 +++++----- pkg/storage/utils/eosfs/eosfs_test.go | 5 ++-- pkg/storage/utils/localfs/localfs.go | 7 +++--- pkg/storage/utils/localfs/upload.go | 6 ++--- pkg/user/manager/nextcloud/nextcloud_test.go | 9 ++++---- tests/integration/grpc/ocm_invitation_test.go | 13 ++++++----- tests/integration/grpc/userprovider_test.go | 9 ++++---- 64 files changed, 288 insertions(+), 235 deletions(-) diff --git a/cmd/reva/download.go b/cmd/reva/download.go index 99eb09ccd0..1e2eabf574 100644 --- a/cmd/reva/download.go +++ b/cmd/reva/download.go @@ -29,7 +29,8 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" @@ -182,7 +183,7 @@ func checkDownloadWebdavRef(protocols []*gateway.FileDownloadProtocol) (io.Reade } c := gowebdav.NewClient(p.DownloadEndpoint, "", "") - c.SetHeader(ctxpkg.TokenHeader, token) + c.SetHeader(appctx.TokenHeader, token) reader, err := c.ReadStream(filePath) if err != nil { diff --git a/cmd/reva/grpc.go b/cmd/reva/grpc.go index b1a7ad2a86..b3a76d7c8e 100644 --- a/cmd/reva/grpc.go +++ b/cmd/reva/grpc.go @@ -26,7 +26,8 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" ins "google.golang.org/grpc/credentials/insecure" @@ -41,8 +42,8 @@ func getAuthContext() context.Context { log.Println(err) return ctx } - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) return ctx } diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index 4ea46f6d09..eda27893a5 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -35,7 +35,8 @@ import ( // TODO(labkode): this should not come from this package. "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/datagateway" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/crypto" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/utils" @@ -267,7 +268,7 @@ func checkUploadWebdavRef(protocols []*gateway.FileUploadProtocol, md os.FileInf } c := gowebdav.NewClient(p.UploadEndpoint, "", "") - c.SetHeader(ctxpkg.TokenHeader, token) + c.SetHeader(appctx.TokenHeader, token) c.SetHeader("Upload-Length", strconv.FormatInt(md.Size(), 10)) if err = c.WriteStream(filePath, fd, 0700); err != nil { diff --git a/internal/grpc/interceptors/token/token.go b/internal/grpc/interceptors/token/token.go index 2b66a37207..ad1ddde8f1 100644 --- a/internal/grpc/interceptors/token/token.go +++ b/internal/grpc/interceptors/token/token.go @@ -21,7 +21,7 @@ package token import ( "context" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -32,11 +32,11 @@ func NewUnary() grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { md, ok := metadata.FromIncomingContext(ctx) if ok && md != nil { - if val, ok := md[ctxpkg.TokenHeader]; ok { + if val, ok := md[appctx.TokenHeader]; ok { if len(val) > 0 && val[0] != "" { tkn := val[0] - ctx = ctxpkg.ContextSetToken(ctx, tkn) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, tkn) + ctx = appctx.ContextSetToken(ctx, tkn) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, tkn) } } } @@ -54,11 +54,11 @@ func NewStream() grpc.StreamServerInterceptor { md, ok := metadata.FromIncomingContext(ss.Context()) if ok && md != nil { - if val, ok := md[ctxpkg.TokenHeader]; ok { + if val, ok := md[appctx.TokenHeader]; ok { if len(val) > 0 && val[0] != "" { tkn := val[0] - ctx = ctxpkg.ContextSetToken(ctx, tkn) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, tkn) + ctx = appctx.ContextSetToken(ctx, tkn) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, tkn) } } } diff --git a/internal/grpc/interceptors/useragent/useragent.go b/internal/grpc/interceptors/useragent/useragent.go index 3c4636fc20..b81cf7a6f3 100644 --- a/internal/grpc/interceptors/useragent/useragent.go +++ b/internal/grpc/interceptors/useragent/useragent.go @@ -21,7 +21,7 @@ package useragent import ( "context" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -31,8 +31,8 @@ import ( func NewUnary() grpc.UnaryServerInterceptor { interceptor := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { if md, ok := metadata.FromIncomingContext(ctx); ok { - if lst, ok := md[ctxpkg.UserAgentHeader]; ok && len(lst) != 0 { - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.UserAgentHeader, lst[0]) + if lst, ok := md[appctx.UserAgentHeader]; ok && len(lst) != 0 { + ctx = metadata.AppendToOutgoingContext(ctx, appctx.UserAgentHeader, lst[0]) } } return handler(ctx, req) @@ -46,8 +46,8 @@ func NewStream() grpc.StreamServerInterceptor { interceptor := func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() if md, ok := metadata.FromIncomingContext(ctx); ok { - if lst, ok := md[ctxpkg.UserAgentHeader]; ok && len(lst) != 0 { - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.UserAgentHeader, lst[0]) + if lst, ok := md[appctx.UserAgentHeader]; ok && len(lst) != 0 { + ctx = metadata.AppendToOutgoingContext(ctx, appctx.UserAgentHeader, lst[0]) } } wrapped := newWrappedServerStream(ctx, ss) diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index 497820bc1f..bb202a7783 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -24,7 +24,7 @@ import ( ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + txdriver "github.com/cs3org/reva/pkg/datatx" txregistry "github.com/cs3org/reva/pkg/datatx/manager/registry" repoRegistry "github.com/cs3org/reva/pkg/datatx/repository/registry" @@ -126,7 +126,7 @@ func (s *service) CreateTransfer(ctx context.Context, req *datatx.CreateTransfer // we always save the transfer regardless of start transfer outcome // only then, if starting fails, can we try to restart it - userID := ctxpkg.ContextMustGetUser(ctx).GetId() + userID := appctx.ContextMustGetUser(ctx).GetId() transfer := &txdriver.Transfer{ TxID: txInfo.GetId().OpaqueId, SrcTargetURI: req.SrcTargetUri, @@ -219,7 +219,7 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer } func (s *service) ListTransfers(ctx context.Context, req *datatx.ListTransfersRequest) (*datatx.ListTransfersResponse, error) { - userID := ctxpkg.ContextMustGetUser(ctx).GetId() + userID := appctx.ContextMustGetUser(ctx).GetId() transfers, err := s.storageDriver.ListTransfers(req.Filters, userID) if err != nil { err = errors.Wrap(err, "datatx service: error listing transfers") diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index 8cce496f46..50e974eeb6 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -30,7 +30,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -318,7 +318,7 @@ func (s *svc) handleTransfer(ctx context.Context, share *ocm.ReceivedShare, tran // http://...token...@reva.eu/prefix/?name=remote.php/webdav/home/... destEndpointPath := strings.TrimPrefix(destWebdavEndpointURL.Path, dstWebdavHostURL.Path) destEndpointScheme := destWebdavEndpointURL.Scheme - destToken := ctxpkg.ContextMustGetToken(ctx) + destToken := appctx.ContextMustGetToken(ctx) destPath := path.Join(destEndpointPath, transferDestinationPath, path.Base(share.Name)) destTargetURI := fmt.Sprintf("%s://%s@%s?name=%s", destEndpointScheme, destToken, destServiceHost, destPath) // var destUri string diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 84d5a554aa..34cc89fa21 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -34,7 +34,7 @@ import ( registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -1775,7 +1775,7 @@ func (s *svc) listSharesFolder(ctx context.Context) (*provider.ListContainerResp } func (s *svc) filterProvidersByUserAgent(ctx context.Context, providers []*registry.ProviderInfo) []*registry.ProviderInfo { - cat, ok := ctxpkg.ContextGetUserAgentCategory(ctx) + cat, ok := appctx.ContextGetUserAgentCategory(ctx) if !ok { return providers } diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index af6dec5ce8..6914627fc2 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -29,7 +29,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -321,8 +321,8 @@ func (s *svc) UpdateReceivedShare(ctx context.Context, req *collaboration.Update // if we don't need to create/delete references then we return early. if !s.c.CommitShareToStorageRef || - ctxpkg.ContextMustGetUser(ctx).Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT || - ctxpkg.ContextMustGetUser(ctx).Id.Type == userpb.UserType_USER_TYPE_FEDERATED { + appctx.ContextMustGetUser(ctx).Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT || + appctx.ContextMustGetUser(ctx).Id.Type == userpb.UserType_USER_TYPE_FEDERATED { return res, nil } diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go index 33c6bb6601..b5e82718c9 100644 --- a/internal/grpc/services/gateway/webdavstorageprovider.go +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -28,7 +28,8 @@ import ( ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/pkg/errors" "github.com/studio-b12/gowebdav" @@ -56,7 +57,7 @@ func (s *svc) webdavRefStat(ctx context.Context, targetURL string, nameQueries . } c := gowebdav.NewClient(webdavEP, "", "") - c.SetHeader(ctxpkg.TokenHeader, ep.token) + c.SetHeader(appctx.TokenHeader, ep.token) // TODO(ishank011): We need to call PROPFIND ourselves as we need to retrieve // ownloud-specific fields to get the resource ID and permissions. @@ -83,7 +84,7 @@ func (s *svc) webdavRefLs(ctx context.Context, targetURL string, nameQueries ... } c := gowebdav.NewClient(webdavEP, "", "") - c.SetHeader(ctxpkg.TokenHeader, ep.token) + c.SetHeader(appctx.TokenHeader, ep.token) // TODO(ishank011): We need to call PROPFIND ourselves as we need to retrieve // ownloud-specific fields to get the resource ID and permissions. @@ -116,7 +117,7 @@ func (s *svc) webdavRefMkdir(ctx context.Context, targetURL string, nameQueries } c := gowebdav.NewClient(webdavEP, "", "") - c.SetHeader(ctxpkg.TokenHeader, ep.token) + c.SetHeader(appctx.TokenHeader, ep.token) err = c.Mkdir(ep.filePath, 0700) if err != nil { @@ -149,7 +150,7 @@ func (s *svc) webdavRefMove(ctx context.Context, targetURL, src, destination str } c := gowebdav.NewClient(srcWebdavEP, "", "") - c.SetHeader(ctxpkg.TokenHeader, srcEP.token) + c.SetHeader(appctx.TokenHeader, srcEP.token) err = c.Rename(srcEP.filePath, destEP.filePath, true) if err != nil { @@ -174,7 +175,7 @@ func (s *svc) webdavRefDelete(ctx context.Context, targetURL string, nameQueries } c := gowebdav.NewClient(webdavEP, "", "") - c.SetHeader(ctxpkg.TokenHeader, ep.token) + c.SetHeader(appctx.TokenHeader, ep.token) err = c.Remove(ep.filePath) if err != nil { diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 76d91111ed..c713affab6 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -35,7 +35,8 @@ import ( providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/ocmd" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/share" @@ -281,7 +282,7 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq } info := statRes.Info - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) tkn := utils.RandString(32) now := time.Now().UnixNano() ts := &typespb.Timestamp{ @@ -373,7 +374,7 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq func (s *service) RemoveOCMShare(ctx context.Context, req *ocm.RemoveOCMShareRequest) (*ocm.RemoveOCMShareResponse, error) { // TODO (gdelmont): notify the remote provider using the /notification ocm endpoint // https://cs3org.github.io/OCM-API/docs.html?branch=develop&repo=OCM-API&user=cs3org#/paths/~1notifications/post - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if err := s.repo.DeleteShare(ctx, user, req.Ref); err != nil { if errors.Is(err, share.ErrShareNotFound) { return &ocm.RemoveOCMShareResponse{ @@ -394,7 +395,7 @@ func (s *service) GetOCMShare(ctx context.Context, req *ocm.GetOCMShareRequest) // if the request is by token, the user does not need to be in the ctx var user *userpb.User if req.Ref.GetToken() == "" { - user = ctxpkg.ContextMustGetUser(ctx) + user = appctx.ContextMustGetUser(ctx) } ocmshare, err := s.repo.GetShare(ctx, user, req.Ref) if err != nil { @@ -438,7 +439,7 @@ func (s *service) GetOCMShareByToken(ctx context.Context, req *ocm.GetOCMShareBy } func (s *service) ListOCMShares(ctx context.Context, req *ocm.ListOCMSharesRequest) (*ocm.ListOCMSharesResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) shares, err := s.repo.ListShares(ctx, user, req.Filters) if err != nil { return &ocm.ListOCMSharesResponse{ @@ -454,7 +455,7 @@ func (s *service) ListOCMShares(ctx context.Context, req *ocm.ListOCMSharesReque } func (s *service) UpdateOCMShare(ctx context.Context, req *ocm.UpdateOCMShareRequest) (*ocm.UpdateOCMShareResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if len(req.Field) == 0 { return &ocm.UpdateOCMShareResponse{ Status: status.NewOK(ctx), @@ -479,7 +480,7 @@ func (s *service) UpdateOCMShare(ctx context.Context, req *ocm.UpdateOCMShareReq } func (s *service) ListReceivedOCMShares(ctx context.Context, req *ocm.ListReceivedOCMSharesRequest) (*ocm.ListReceivedOCMSharesResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) shares, err := s.repo.ListReceivedShares(ctx, user) if err != nil { return &ocm.ListReceivedOCMSharesResponse{ @@ -495,7 +496,7 @@ func (s *service) ListReceivedOCMShares(ctx context.Context, req *ocm.ListReceiv } func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceivedOCMShareRequest) (*ocm.UpdateReceivedOCMShareResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) _, err := s.repo.UpdateReceivedShare(ctx, user, req.Share, req.UpdateMask) if err != nil { if errors.Is(err, share.ErrShareNotFound) { @@ -515,7 +516,7 @@ func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateRec } func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMShareRequest) (*ocm.GetReceivedOCMShareResponse, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) ocmshare, err := s.repo.GetReceivedShare(ctx, user, req.Ref) if err != nil { if errors.Is(err, share.ErrShareNotFound) { diff --git a/internal/grpc/services/pingpong/proto/pingpong.pb.go b/internal/grpc/services/pingpong/proto/pingpong.pb.go index aac5826968..bd2c4b37f8 100644 --- a/internal/grpc/services/pingpong/proto/pingpong.pb.go +++ b/internal/grpc/services/pingpong/proto/pingpong.pb.go @@ -25,10 +25,11 @@ package proto import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go b/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go index 26f6e066a6..31a0f3c7ab 100644 --- a/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go +++ b/internal/grpc/services/pingpong/proto/pingpong_grpc.pb.go @@ -8,6 +8,7 @@ package proto import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index 75154e3b7b..6062c52397 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -33,7 +33,7 @@ import ( tokenregistry "github.com/cs3org/reva/internal/http/interceptors/auth/token/registry" tokenwriterregistry "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/registry" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" @@ -191,7 +191,7 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke log := appctx.GetLogger(ctx) // Add the request user-agent to the ctx - ctx = metadata.NewIncomingContext(ctx, metadata.New(map[string]string{ctxpkg.UserAgentHeader: r.UserAgent()})) + ctx = metadata.NewIncomingContext(ctx, metadata.New(map[string]string{appctx.UserAgentHeader: r.UserAgent()})) client, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySvc)) if err != nil { @@ -297,10 +297,10 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke } func ctxWithUserInfo(ctx context.Context, r *http.Request, user *userpb.User, token string) context.Context { - ctx = ctxpkg.ContextSetUser(ctx, user) - ctx = ctxpkg.ContextSetToken(ctx, token) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, token) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.UserAgentHeader, r.UserAgent()) + ctx = appctx.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, token) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.UserAgentHeader, r.UserAgent()) return ctx } diff --git a/internal/http/interceptors/auth/token/strategy/header/header.go b/internal/http/interceptors/auth/token/strategy/header/header.go index 3f1c30bcf7..f25e55a8f6 100644 --- a/internal/http/interceptors/auth/token/strategy/header/header.go +++ b/internal/http/interceptors/auth/token/strategy/header/header.go @@ -22,7 +22,8 @@ import ( "net/http" "github.com/cs3org/reva/internal/http/interceptors/auth/token/registry" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" ) @@ -36,7 +37,7 @@ type strategy struct { // New returns a new auth strategy that checks for basic auth. func New(m map[string]interface{}) (auth.TokenStrategy, error) { - return &strategy{header: ctxpkg.TokenHeader}, nil + return &strategy{header: appctx.TokenHeader}, nil } func (s *strategy) GetToken(r *http.Request) string { diff --git a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go index d3e689ec53..24a3fc3551 100644 --- a/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go +++ b/internal/http/interceptors/auth/tokenwriter/strategy/header/header.go @@ -22,7 +22,8 @@ import ( "net/http" "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/registry" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" ) @@ -36,7 +37,7 @@ type strategy struct { // New returns a new token writer strategy that stores token in a header. func New(m map[string]interface{}) (auth.TokenWriter, error) { - return &strategy{header: ctxpkg.TokenHeader}, nil + return &strategy{header: appctx.TokenHeader}, nil } func (s *strategy) WriteToken(token string, w http.ResponseWriter) { diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index b19a3d870d..e6916e81fd 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -35,7 +35,7 @@ import ( storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/global" @@ -164,7 +164,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { return } - token, ok := ctxpkg.ContextGetToken(ctx) + token, ok := appctx.ContextGetToken(ctx) if !ok || token == "" { reqres.WriteError(w, r, reqres.APIErrorUnauthenticated, "Access token is invalid or empty", err) return @@ -195,7 +195,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { return } - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) scope, err := scope.AddResourceInfoScope(resource, authpb.Role_ROLE_VIEWER, nil) if err != nil { diff --git a/internal/http/services/owncloud/ocdav/dav.go b/internal/http/services/owncloud/ocdav/dav.go index 07327337ca..a4ff90430b 100644 --- a/internal/http/services/owncloud/ocdav/dav.go +++ b/internal/http/services/owncloud/ocdav/dav.go @@ -30,7 +30,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" "google.golang.org/grpc/metadata" @@ -107,7 +107,7 @@ func (h *DavHandler) Handler(s *svc) http.Handler { // https://github.com/owncloud/core/blob/18475dac812064b21dabcc50f25ef3ffe55691a5/tests/acceptance/features/apiWebdavOperations/propfind.feature if r.URL.Path == "/files" { log.Debug().Str("path", r.URL.Path).Msg("method not allowed") - contextUser, ok := ctxpkg.ContextGetUser(ctx) + contextUser, ok := appctx.ContextGetUser(ctx) if ok { r.URL.Path = path.Join(r.URL.Path, contextUser.Username) } @@ -147,7 +147,7 @@ func (h *DavHandler) Handler(s *svc) http.Handler { requestUserID, r.URL.Path = router.ShiftPath(r.URL.Path) // note: some requests like OPTIONS don't forward the user - contextUser, ok := ctxpkg.ContextGetUser(ctx) + contextUser, ok := appctx.ContextGetUser(ctx) if ok && isOwner(requestUserID, contextUser) { // use home storage handler when user was detected base := path.Join(ctx.Value(ctxKeyBaseURI).(string), "files", requestUserID) @@ -225,9 +225,9 @@ func (h *DavHandler) Handler(s *svc) http.Handler { return } - ctx = ctxpkg.ContextSetToken(ctx, authRes.Token) - ctx = ctxpkg.ContextSetUser(ctx, authRes.User) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, authRes.Token) + ctx = appctx.ContextSetToken(ctx, authRes.Token) + ctx = appctx.ContextSetUser(ctx, authRes.User) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, authRes.Token) log.Debug().Str("token", token).Interface("user", authRes.User).Msg("OCM user authenticated") @@ -274,9 +274,9 @@ func (h *DavHandler) Handler(s *svc) http.Handler { return } - ctx = ctxpkg.ContextSetToken(ctx, res.Token) - ctx = ctxpkg.ContextSetUser(ctx, res.User) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, res.Token) + ctx = appctx.ContextSetToken(ctx, res.Token) + ctx = appctx.ContextSetUser(ctx, res.User) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, res.Token) r = r.WithContext(ctx) diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 12e2f85235..6606be76f5 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -32,7 +32,7 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/notification/notificationhelper" @@ -291,7 +291,7 @@ func applyLayout(ctx context.Context, ns string, useLoggedInUserNS bool, request // is not the same as the logged in user. In that case, we'll treat fileOwner // as the username whose files are to be accessed and use that in the // namespace template. - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok || !useLoggedInUserNS { requestUserID, _ := router.ShiftPath(requestPath) u = &userpb.User{ diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index b91685d916..ab6e341623 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -42,7 +42,7 @@ import ( "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/share" @@ -155,7 +155,7 @@ func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *ht listResp, err := client.ListPublicShares(ctx, &link.ListPublicSharesRequest{ Opaque: &types.Opaque{ Map: map[string]*types.OpaqueEntry{ - ctxpkg.ResoucePathCtx: {Decoder: "plain", Value: []byte(parentInfo.Path)}, + appctx.ResoucePathCtx: {Decoder: "plain", Value: []byte(parentInfo.Path)}, }, }, Filters: linkFilters, @@ -174,7 +174,7 @@ func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *ht Filters: shareFilters, Opaque: &types.Opaque{ Map: map[string]*types.OpaqueEntry{ - ctxpkg.ResoucePathCtx: {Decoder: "plain", Value: []byte(parentInfo.Path)}, + appctx.ResoucePathCtx: {Decoder: "plain", Value: []byte(parentInfo.Path)}, }, }, }) @@ -734,10 +734,10 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide case "public-link-share-owner": if ls != nil && ls.Owner != nil { if isCurrentUserOwner(ctx, ls.Owner) { - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:public-link-share-owner", u.Username)) } else { - u, _ := ctxpkg.ContextGetUser(ctx) + u, _ := appctx.ContextGetUser(ctx) sublog.Error().Interface("share", ls).Interface("user", u).Msg("the current user in the context should be the owner of a public link share") propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:public-link-share-owner", "")) } @@ -766,7 +766,7 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide case "owner-id": // phoenix only if md.Owner != nil { if isCurrentUserOwner(ctx, md.Owner) { - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:owner-id", u.Username)) } else { sublog.Debug().Msg("TODO fetch user username") @@ -855,7 +855,7 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide case "owner-display-name": // phoenix only if md.Owner != nil { if isCurrentUserOwner(ctx, md.Owner) { - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:owner-display-name", u.DisplayName)) } else { sublog.Debug().Msg("TODO fetch user displayname") @@ -1036,7 +1036,7 @@ func quoteEtag(etag string) string { // a file is only yours if you are the owner. func isCurrentUserOwner(ctx context.Context, owner *userv1beta1.UserId) bool { - contextUser, ok := ctxpkg.ContextGetUser(ctx) + contextUser, ok := appctx.ContextGetUser(ctx) if ok && contextUser.Id != nil && owner != nil && contextUser.Id.Idp == owner.Idp && contextUser.Id.OpaqueId == owner.OpaqueId { diff --git a/internal/http/services/owncloud/ocdav/proppatch.go b/internal/http/services/owncloud/ocdav/proppatch.go index 3d0c91ee41..abe928ede8 100644 --- a/internal/http/services/owncloud/ocdav/proppatch.go +++ b/internal/http/services/owncloud/ocdav/proppatch.go @@ -30,7 +30,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/pkg/errors" "github.com/rs/zerolog" ) @@ -234,7 +234,7 @@ func (s *svc) handleProppatch(ctx context.Context, w http.ResponseWriter, r *htt w.WriteHeader(http.StatusInternalServerError) return nil, nil, false } - currentUser := ctxpkg.ContextMustGetUser(ctx) + currentUser := appctx.ContextMustGetUser(ctx) err = s.favoritesManager.UnsetFavorite(ctx, currentUser.Id, statRes.Info) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -275,7 +275,7 @@ func (s *svc) handleProppatch(ctx context.Context, w http.ResponseWriter, r *htt w.WriteHeader(http.StatusInternalServerError) return nil, nil, false } - currentUser := ctxpkg.ContextMustGetUser(ctx) + currentUser := appctx.ContextMustGetUser(ctx) err = s.favoritesManager.SetFavorite(ctx, currentUser.Id, statRes.Info) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/http/services/owncloud/ocdav/report.go b/internal/http/services/owncloud/ocdav/report.go index 837235a4fd..b77d967865 100644 --- a/internal/http/services/owncloud/ocdav/report.go +++ b/internal/http/services/owncloud/ocdav/report.go @@ -27,7 +27,6 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" ) const ( @@ -79,7 +78,7 @@ func (s *svc) doFilterFiles(w http.ResponseWriter, r *http.Request, ff *reportFi if ff.Rules.Favorite { // List the users favorite resources. - currentUser := ctxpkg.ContextMustGetUser(ctx) + currentUser := appctx.ContextMustGetUser(ctx) favorites, err := s.favoritesManager.ListFavorites(ctx, currentUser.Id) if err != nil { log.Error().Err(err).Msg("error getting favorites") diff --git a/internal/http/services/owncloud/ocdav/trashbin.go b/internal/http/services/owncloud/ocdav/trashbin.go index 3c24afbb1e..bcc94cc7eb 100644 --- a/internal/http/services/owncloud/ocdav/trashbin.go +++ b/internal/http/services/owncloud/ocdav/trashbin.go @@ -33,7 +33,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/utils" @@ -70,7 +70,7 @@ func (h *TrashbinHandler) Handler(s *svc) http.Handler { return } - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { w.WriteHeader(http.StatusBadRequest) return diff --git a/internal/http/services/owncloud/ocs/cache.go b/internal/http/services/owncloud/ocs/cache.go index 56cb588ba4..ce231b1816 100644 --- a/internal/http/services/owncloud/ocs/cache.go +++ b/internal/http/services/owncloud/ocs/cache.go @@ -24,14 +24,14 @@ import ( "net/http/httptest" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "google.golang.org/grpc/metadata" ) func (s *svc) cacheWarmup(w http.ResponseWriter, r *http.Request) { if s.warmupCacheTracker != nil { - u, ok1 := ctxpkg.ContextGetUser(r.Context()) - tkn, ok2 := ctxpkg.ContextGetToken(r.Context()) + u, ok1 := appctx.ContextGetUser(r.Context()) + tkn, ok2 := appctx.ContextGetToken(r.Context()) if !ok1 || !ok2 { return } @@ -44,9 +44,9 @@ func (s *svc) cacheWarmup(w http.ResponseWriter, r *http.Request) { // TODO: Check if we can come up with a better solution, eg, https://stackoverflow.com/a/54132324 ctx := context.Background() ctx = appctx.WithLogger(ctx, log) - ctx = ctxpkg.ContextSetUser(ctx, u) - ctx = ctxpkg.ContextSetToken(ctx, tkn) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, tkn) + ctx = appctx.ContextSetUser(ctx, u) + ctx = appctx.ContextSetToken(ctx, tkn) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, tkn) req, _ := http.NewRequest(http.MethodGet, "", nil) req = req.WithContext(ctx) diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go index 96f6acf0b9..f77febfbe2 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go @@ -29,7 +29,8 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) @@ -85,7 +86,7 @@ func (h *Handler) createGroupShare(w http.ResponseWriter, r *http.Request, statI if shareID, ok := h.createCs3Share(ctx, w, r, c, createShareReq, statInfo); ok { notify, _ := strconv.ParseBool(r.FormValue("notify")) if notify { - granter, ok := ctxpkg.ContextGetUser(ctx) + granter, ok := appctx.ContextGetUser(ctx) if ok { h.SendShareNotification(shareID.OpaqueId, granter, groupRes.Group, statInfo) } diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go index 8b0662a7de..f98169c05d 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -421,7 +421,7 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar if ok { ok2 := permissionsStayUploader(before, newPermissions) - u, ok3 := ctxpkg.ContextGetUser(r.Context()) + u, ok3 := appctx.ContextGetUser(r.Context()) if ok2 && ok3 { notifyUploads, _ := strconv.ParseBool(newNotifyUploads[0]) @@ -451,7 +451,7 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar if ok { ok2 := permissionsStayUploader(before, newPermissions) - u, ok3 := ctxpkg.ContextGetUser(r.Context()) + u, ok3 := appctx.ContextGetUser(r.Context()) if ok2 && ok3 { notifyUploadsExtraRecipients := newNotifyUploadsExtraRecipients[0] diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 5c042a475d..79ee760c23 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -48,7 +48,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/notification/notificationhelper" "github.com/cs3org/reva/pkg/notification/trigger" @@ -271,7 +271,7 @@ func (h *Handler) NotifyShare(w http.ResponseWriter, r *http.Request) { return } - granter, ok := ctxpkg.ContextGetUser(ctx) + granter, ok := appctx.ContextGetUser(ctx) if !ok { h.Log.Error().Err(err).Msgf("error getting granter data") response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting granter data", err) diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go index 255e5d2f8b..ebccf1cd0d 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) @@ -91,7 +91,7 @@ func (h *Handler) createUserShare(w http.ResponseWriter, r *http.Request, statIn if shareID, ok := h.createCs3Share(ctx, w, r, c, createShareReq, statInfo); ok { notify, _ := strconv.ParseBool(r.FormValue("notify")) if notify { - granter, ok := ctxpkg.ContextGetUser(ctx) + granter, ok := appctx.ContextGetUser(ctx) if ok { h.SendShareNotification(shareID.OpaqueId, granter, userRes.User, statInfo) } @@ -287,7 +287,7 @@ func (h *Handler) listUserShares(r *http.Request, filters []*collaboration.Filte Filters: filters, Opaque: &types.Opaque{ Map: map[string]*types.OpaqueEntry{ - ctxpkg.ResoucePathCtx: {Decoder: "plain", Value: []byte(ctxPath)}, + appctx.ResoucePathCtx: {Decoder: "plain", Value: []byte(ctxPath)}, }, }, } diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go b/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go index c554be8a3c..b717ca38e5 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/uploads.go @@ -23,7 +23,8 @@ import ( "strings" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/data" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" + "github.com/juliangruber/go-intersect" ) @@ -58,7 +59,7 @@ func (h *Handler) getCapabilitiesForUserAgent(ctx context.Context, userAgent str } func ctxUserBelongsToGroups(ctx context.Context, groups []string) bool { - if user, ok := ctxpkg.ContextGetUser(ctx); ok { + if user, ok := appctx.ContextGetUser(ctx); ok { return len(intersect.Simple(groups, user.Groups)) > 0 } return false diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go b/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go index cc33bb370f..12ba0b013d 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go @@ -30,7 +30,8 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/config" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) @@ -59,7 +60,7 @@ func (h *Handler) GetSelf(w http.ResponseWriter, r *http.Request) { ctx := r.Context() // TODO move user to handler parameter? - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "missing user in context", fmt.Errorf("missing user in context")) return diff --git a/internal/http/services/pingpong/pingpong.go b/internal/http/services/pingpong/pingpong.go index 31739dd01f..b88d933a99 100644 --- a/internal/http/services/pingpong/pingpong.go +++ b/internal/http/services/pingpong/pingpong.go @@ -128,11 +128,13 @@ func (s *svc) doPing(w http.ResponseWriter, r *http.Request) { } log.Info().Msg("pinging from http to grpc") - - w.Write([]byte(pingRes.Info)) + _, err = w.Write([]byte(pingRes.Info)) + log.Error().Err(err).Msg("error writing res") } // doPong will be (http) called from grpc Pong. func (s *svc) doPong(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("pong")) + log := appctx.GetLogger(r.Context()) + _, err := w.Write([]byte("pong")) + log.Error().Err(err).Msg("error writing res") } diff --git a/internal/http/services/reverseproxy/reverseproxy.go b/internal/http/services/reverseproxy/reverseproxy.go index 50c53f564e..96b513c659 100644 --- a/internal/http/services/reverseproxy/reverseproxy.go +++ b/internal/http/services/reverseproxy/reverseproxy.go @@ -26,7 +26,7 @@ import ( "net/url" "os" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" @@ -85,8 +85,8 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) proxy := httputil.NewSingleHostReverseProxy(remote) handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Host = remote.Host - if token, ok := ctxpkg.ContextGetToken(r.Context()); ok { - r.Header.Set(ctxpkg.TokenHeader, token) + if token, ok := appctx.ContextGetToken(r.Context()); ok { + r.Header.Set(appctx.TokenHeader, token) } proxy.ServeHTTP(w, r) }) diff --git a/internal/http/services/sciencemesh/token.go b/internal/http/services/sciencemesh/token.go index 54f97ea752..630c240203 100644 --- a/internal/http/services/sciencemesh/token.go +++ b/internal/http/services/sciencemesh/token.go @@ -32,7 +32,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/smtpclient" "github.com/cs3org/reva/pkg/utils/list" @@ -94,7 +94,7 @@ func (h *tokenHandler) Generate(w http.ResponseWriter, r *http.Request) { return } - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) recipient := query.Get("recipient") if recipient != "" && h.smtpCredentials != nil { templObj := &emailParams{ diff --git a/pkg/appauth/manager/json/json.go b/pkg/appauth/manager/json/json.go index 650e1ba742..0ec6f48e95 100644 --- a/pkg/appauth/manager/json/json.go +++ b/pkg/appauth/manager/json/json.go @@ -32,7 +32,8 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appauth" "github.com/cs3org/reva/pkg/appauth/manager/registry" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" @@ -127,7 +128,7 @@ func (mgr *jsonManager) GenerateAppPassword(ctx context.Context, scope map[strin if err != nil { return nil, errors.Wrap(err, "error creating new token") } - userID := ctxpkg.ContextMustGetUser(ctx).GetId() + userID := appctx.ContextMustGetUser(ctx).GetId() ctime := now() password := string(tokenHashed) @@ -161,7 +162,7 @@ func (mgr *jsonManager) GenerateAppPassword(ctx context.Context, scope map[strin } func (mgr *jsonManager) ListAppPasswords(ctx context.Context) ([]*apppb.AppPassword, error) { - userID := ctxpkg.ContextMustGetUser(ctx).GetId() + userID := appctx.ContextMustGetUser(ctx).GetId() mgr.Lock() defer mgr.Unlock() appPasswords := []*apppb.AppPassword{} @@ -172,7 +173,7 @@ func (mgr *jsonManager) ListAppPasswords(ctx context.Context) ([]*apppb.AppPassw } func (mgr *jsonManager) InvalidateAppPassword(ctx context.Context, password string) error { - userID := ctxpkg.ContextMustGetUser(ctx).GetId() + userID := appctx.ContextMustGetUser(ctx).GetId() mgr.Lock() defer mgr.Unlock() diff --git a/pkg/appauth/manager/json/json_test.go b/pkg/appauth/manager/json/json_test.go index 14b0b97c8f..d2050305d0 100644 --- a/pkg/appauth/manager/json/json_test.go +++ b/pkg/appauth/manager/json/json_test.go @@ -32,7 +32,8 @@ import ( apppb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" + "github.com/gdexlab/go-render/render" "github.com/sethvargo/go-password/password" "golang.org/x/crypto/bcrypt" @@ -140,7 +141,7 @@ func TestNewManager(t *testing.T) { func TestGenerateAppPassword(t *testing.T) { userTest := &userpb.User{Id: &userpb.UserId{Idp: "0"}, Username: "Test User"} - ctx := ctxpkg.ContextSetUser(context.Background(), userTest) + ctx := appctx.ContextSetUser(context.Background(), userTest) tempDir := createTempDir(t, "jsonappauth_test") defer os.RemoveAll(tempDir) @@ -290,7 +291,7 @@ func TestGenerateAppPassword(t *testing.T) { func TestListAppPasswords(t *testing.T) { user0Test := &userpb.User{Id: &userpb.UserId{Idp: "0"}} user1Test := &userpb.User{Id: &userpb.UserId{Idp: "1"}} - ctx := ctxpkg.ContextSetUser(context.Background(), user0Test) + ctx := appctx.ContextSetUser(context.Background(), user0Test) tempDir := createTempDir(t, "jsonappauth_test") defer os.RemoveAll(tempDir) @@ -415,7 +416,7 @@ func TestListAppPasswords(t *testing.T) { func TestInvalidateAppPassword(t *testing.T) { userTest := &userpb.User{Id: &userpb.UserId{Idp: "0"}} - ctx := ctxpkg.ContextSetUser(context.Background(), userTest) + ctx := appctx.ContextSetUser(context.Background(), userTest) tempDir := createTempDir(t, "jsonappauth_test") defer os.RemoveAll(tempDir) @@ -539,7 +540,7 @@ func TestInvalidateAppPassword(t *testing.T) { func TestGetAppPassword(t *testing.T) { userTest := &userpb.User{Id: &userpb.UserId{Idp: "0"}} - ctx := ctxpkg.ContextSetUser(context.Background(), userTest) + ctx := appctx.ContextSetUser(context.Background(), userTest) tempDir := createTempDir(t, "jsonappauth_test") defer os.RemoveAll(tempDir) diff --git a/pkg/auth/manager/nextcloud/nextcloud_test.go b/pkg/auth/manager/nextcloud/nextcloud_test.go index 26dfe9243c..a09879edba 100644 --- a/pkg/auth/manager/nextcloud/nextcloud_test.go +++ b/pkg/auth/manager/nextcloud/nextcloud_test.go @@ -26,7 +26,8 @@ import ( authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/manager/nextcloud" "github.com/cs3org/reva/pkg/auth/scope" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" @@ -99,9 +100,9 @@ var _ = Describe("Nextcloud", func() { Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) + ctx = appctx.ContextSetUser(ctx, user) }) AfterEach(func() { diff --git a/pkg/cbox/share/sql/sql.go b/pkg/cbox/share/sql/sql.go index e912d20886..6352d35eea 100644 --- a/pkg/cbox/share/sql/sql.go +++ b/pkg/cbox/share/sql/sql.go @@ -32,7 +32,8 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" conversions "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -105,7 +106,7 @@ func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { } func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) // do not allow share to myself or the owner if share is for a user // TODO(labkode): should not this be caught already at the gw level? @@ -176,7 +177,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collabora } func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) { - uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id) + uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) s := conversions.DBShare{ID: id.OpaqueId} query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND id=? AND (uid_owner=? or uid_initiator=?)" if err := m.db.QueryRow(query, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType); err != nil { @@ -191,7 +192,7 @@ func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collabor func (m *mgr) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) { owner := conversions.FormatUserID(key.Owner) - uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id) + uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) s := conversions.DBShare{} shareType, shareWith := conversions.FormatGrantee(key.Grantee) @@ -226,7 +227,7 @@ func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) ( } func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { - uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id) + uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) var query string params := []interface{}{} switch { @@ -264,7 +265,7 @@ func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) er func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { permissions := conversions.SharePermToInt(p.Permissions) - uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id) + uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) var query string params := []interface{}{} @@ -345,7 +346,7 @@ func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ( // we list the shares that are targeted to the user in context or to the user groups. func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) uid := conversions.FormatUserID(user.Id) params := []interface{}{uid, uid, uid, uid} @@ -398,7 +399,7 @@ func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.F } func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.ReceivedShare, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) uid := conversions.FormatUserID(user.Id) params := []interface{}{uid, id.OpaqueId, uid} @@ -428,7 +429,7 @@ func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (* } func (m *mgr) getReceivedByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.ReceivedShare, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) uid := conversions.FormatUserID(user.Id) shareType, shareWith := conversions.FormatGrantee(key.Grantee) @@ -480,7 +481,7 @@ func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareRefe } func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) rs, err := m.GetReceivedShare(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Share.Id}}) if err != nil { @@ -520,7 +521,7 @@ func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.Rece } func (m *mgr) uidOwnerFilters(ctx context.Context, filters map[collaboration.Filter_Type][]*collaboration.Filter) (string, []interface{}, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) uid := conversions.FormatUserID(user.Id) query := "uid_owner=? or uid_initiator=?" diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go index 7fd7eac89c..cee21596a7 100644 --- a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go +++ b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go @@ -25,7 +25,8 @@ import ( "github.com/Masterminds/sprig" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" @@ -105,7 +106,7 @@ func (w *wrapper) DenyGrant(ctx context.Context, ref *provider.Reference, g *pro } func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string { - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) b := bytes.Buffer{} if err := w.mountIDTemplate.Execute(&b, u); err != nil { return "" diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go index 984f35cdd4..80e5f3a0d0 100644 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -27,7 +27,8 @@ import ( "github.com/Masterminds/sprig" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" @@ -184,7 +185,7 @@ func (w *wrapper) setProjectSharingPermissions(ctx context.Context, r *provider. return nil } adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, g := range user.Groups { if g == adminGroup { @@ -220,7 +221,7 @@ func (w *wrapper) userIsProjectAdmin(ctx context.Context, ref *provider.Referenc return nil } adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, g := range user.Groups { if g == adminGroup { diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index fb1be47883..d2b056c16b 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -33,7 +33,7 @@ import ( "time" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/acl" @@ -516,7 +516,7 @@ func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, a func (c *Client) handleFavAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string, info *eosclient.FileInfo, set bool) error { var err error - u := ctxpkg.ContextMustGetUser(ctx) + u := appctx.ContextMustGetUser(ctx) if info == nil { info, err = c.getRawFileInfoByPath(ctx, auth, path) if err != nil { @@ -1272,7 +1272,7 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, func parseAndSetFavoriteAttr(ctx context.Context, attrs map[string]string) { // Read and correctly set the favorite attr - if user, ok := ctxpkg.ContextGetUser(ctx); ok { + if user, ok := appctx.ContextGetUser(ctx); ok { if favAttrStr, ok := attrs[favoritesKey]; ok { favUsers, err := acl.Parse(favAttrStr, acl.ShortTextForm) if err != nil { diff --git a/pkg/httpclient/httpclient.go b/pkg/httpclient/httpclient.go index 5da392a402..2bef4051e2 100644 --- a/pkg/httpclient/httpclient.go +++ b/pkg/httpclient/httpclient.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ // In applying this license, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. + package httpclient import ( diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go index 7b01d9c06b..b04d1067c6 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go @@ -27,7 +27,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" masked_share "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/nextcloud" @@ -102,9 +103,9 @@ var _ = Describe("Nextcloud", func() { Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) + ctx = appctx.ContextSetUser(ctx, user) }) AfterEach(func() { diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 21d443cd8e..dda3ae1e54 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -34,7 +34,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -248,9 +249,9 @@ func (d *driver) opFromUser(ctx context.Context, userID *userv1beta1.UserId, f f } ownerCtx := context.TODO() - ownerCtx = ctxpkg.ContextSetToken(ownerCtx, authRes.Token) - ownerCtx = ctxpkg.ContextSetUser(ownerCtx, authRes.User) - ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) + ownerCtx = appctx.ContextSetToken(ownerCtx, authRes.Token) + ownerCtx = appctx.ContextSetUser(ownerCtx, authRes.User) + ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, appctx.TokenHeader, authRes.Token) return f(ownerCtx) } diff --git a/pkg/preferences/memory/memory.go b/pkg/preferences/memory/memory.go index ed069932f3..90333ea318 100644 --- a/pkg/preferences/memory/memory.go +++ b/pkg/preferences/memory/memory.go @@ -22,7 +22,7 @@ import ( "context" "sync" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" @@ -43,7 +43,7 @@ func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, er } func (m *mgr) SetKey(ctx context.Context, key, namespace, value string) error { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { return errtypes.UserRequired("preferences: error getting user from ctx") } @@ -61,7 +61,7 @@ func (m *mgr) SetKey(ctx context.Context, key, namespace, value string) error { } func (m *mgr) GetKey(ctx context.Context, key, namespace string) (string, error) { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { return "", errtypes.UserRequired("preferences: error getting user from ctx") } diff --git a/pkg/preferences/sql/sql.go b/pkg/preferences/sql/sql.go index c46469245e..57f34878c4 100644 --- a/pkg/preferences/sql/sql.go +++ b/pkg/preferences/sql/sql.go @@ -23,7 +23,7 @@ import ( "database/sql" "fmt" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" @@ -66,7 +66,7 @@ func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, er } func (m *mgr) SetKey(ctx context.Context, key, namespace, value string) error { - user, ok := ctxpkg.ContextGetUser(ctx) + user, ok := appctx.ContextGetUser(ctx) if !ok { return errtypes.UserRequired("preferences: error getting user from ctx") } @@ -84,7 +84,7 @@ func (m *mgr) SetKey(ctx context.Context, key, namespace, value string) error { } func (m *mgr) GetKey(ctx context.Context, key, namespace string) (string, error) { - user, ok := ctxpkg.ContextGetUser(ctx) + user, ok := appctx.ContextGetUser(ctx) if !ok { return "", errtypes.UserRequired("preferences: error getting user from ctx") } diff --git a/pkg/prom/registry/registry.go b/pkg/prom/registry/registry.go index f5b9be6898..f41fd73813 100644 --- a/pkg/prom/registry/registry.go +++ b/pkg/prom/registry/registry.go @@ -1,3 +1,21 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package registry import ( diff --git a/pkg/sdk/common/net/net.go b/pkg/sdk/common/net/net.go index 7db9c5056b..38fab412b1 100644 --- a/pkg/sdk/common/net/net.go +++ b/pkg/sdk/common/net/net.go @@ -20,7 +20,7 @@ package net import ( "github.com/cs3org/reva/internal/http/services/datagateway" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" ) type ctxKey int @@ -29,7 +29,7 @@ const ( // AccessTokenIndex specifies the index of the Reva access token in a context. AccessTokenIndex ctxKey = iota // AccessTokenName specifies the name of the Reva access token used during requests. - AccessTokenName = ctxpkg.TokenHeader + AccessTokenName = appctx.TokenHeader // TransportTokenName specifies the name of the Reva transport token used during data transfers. TransportTokenName = datagateway.TokenTransportHeader ) diff --git a/pkg/share/cache/warmup/cbox/cbox.go b/pkg/share/cache/warmup/cbox/cbox.go index 10e031693d..b1876cfa22 100644 --- a/pkg/share/cache/warmup/cbox/cbox.go +++ b/pkg/share/cache/warmup/cbox/cbox.go @@ -26,7 +26,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share/cache" @@ -117,7 +118,7 @@ func (m *manager) GetResourceInfos() ([]*provider.ResourceInfo, error) { if err != nil { return nil, err } - ctx := metadata.AppendToOutgoingContext(context.Background(), ctxpkg.TokenHeader, tkn) + ctx := metadata.AppendToOutgoingContext(context.Background(), appctx.TokenHeader, tkn) client, err := pool.GetGatewayServiceClient(pool.Endpoint(m.conf.GatewaySvc)) if err != nil { diff --git a/pkg/share/manager/json/json.go b/pkg/share/manager/json/json.go index 4646fbb8f6..6f6c3a28cc 100644 --- a/pkg/share/manager/json/json.go +++ b/pkg/share/manager/json/json.go @@ -29,7 +29,8 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" @@ -166,7 +167,7 @@ func genID() string { func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { id := genID() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) now := time.Now().UnixNano() ts := &typespb.Timestamp{ Seconds: uint64(now / 1000000000), @@ -256,7 +257,7 @@ func (m *mgr) get(ctx context.Context, ref *collaboration.ShareReference) (s *co } // check if we are the owner - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if share.IsCreatedByUser(s, user) { return s, nil } @@ -282,7 +283,7 @@ func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) ( func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for i, s := range m.model.Shares { if sharesEqual(ref, s) { if share.IsCreatedByUser(s, user) { @@ -316,7 +317,7 @@ func sharesEqual(ref *collaboration.ShareReference, s *collaboration.Share) bool func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for i, s := range m.model.Shares { if sharesEqual(ref, s) { if share.IsCreatedByUser(s, user) { @@ -341,7 +342,7 @@ func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ( var ss []*collaboration.Share m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.model.Shares { if share.IsCreatedByUser(s, user) { // no filter we return earlier @@ -363,7 +364,7 @@ func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.F var rss []*collaboration.ReceivedShare m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.model.Shares { if share.IsCreatedByUser(s, user) || !share.IsGrantedToUser(s, user) { // omit shares created by the user or shares the user can't access @@ -390,7 +391,7 @@ func (m *mgr) convert(ctx context.Context, s *collaboration.Share) *collaboratio Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if v, ok := m.model.State[user.Id.String()]; ok { if state, ok := v[s.Id.String()]; ok { rs.State = state @@ -406,7 +407,7 @@ func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareRefe func (m *mgr) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.model.Shares { if sharesEqual(ref, s) { if share.IsGrantedToUser(s, user) { @@ -424,7 +425,7 @@ func (m *mgr) UpdateReceivedShare(ctx context.Context, receivedShare *collaborat return nil, err } - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) m.Lock() defer m.Unlock() diff --git a/pkg/share/manager/memory/memory.go b/pkg/share/manager/memory/memory.go index 93094b759f..844e69e686 100644 --- a/pkg/share/manager/memory/memory.go +++ b/pkg/share/manager/memory/memory.go @@ -29,7 +29,8 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" @@ -68,7 +69,7 @@ func (m *manager) add(ctx context.Context, s *collaboration.Share) { func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { id := atomic.AddUint64(&counter, 1) - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) now := time.Now().UnixNano() ts := &typespb.Timestamp{ Seconds: uint64(now / 1000000000), @@ -147,7 +148,7 @@ func (m *manager) get(ctx context.Context, ref *collaboration.ShareReference) (s } // check if we are the owner - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if share.IsCreatedByUser(s, user) { return s, nil } @@ -168,7 +169,7 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.lock.Lock() defer m.lock.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for i, s := range m.shares { if sharesEqual(ref, s) { if share.IsCreatedByUser(s, user) { @@ -198,7 +199,7 @@ func sharesEqual(ref *collaboration.ShareReference, s *collaboration.Share) bool func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.lock.Lock() defer m.lock.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for i, s := range m.shares { if sharesEqual(ref, s) { if share.IsCreatedByUser(s, user) { @@ -219,7 +220,7 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte var ss []*collaboration.Share m.lock.Lock() defer m.lock.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.shares { if share.IsCreatedByUser(s, user) { // no filter we return earlier @@ -241,7 +242,7 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati var rss []*collaboration.ReceivedShare m.lock.Lock() defer m.lock.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.shares { if share.IsCreatedByUser(s, user) || !share.IsGrantedToUser(s, user) { // omit shares created by the user or shares the user can't access @@ -268,7 +269,7 @@ func (m *manager) convert(ctx context.Context, s *collaboration.Share) *collabor Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) if v, ok := m.shareState[user.Id.String()]; ok { if state, ok := v[s.Id]; ok { rs.State = state @@ -284,7 +285,7 @@ func (m *manager) GetReceivedShare(ctx context.Context, ref *collaboration.Share func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.lock.Lock() defer m.lock.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) for _, s := range m.shares { if sharesEqual(ref, s) { if share.IsGrantedToUser(s, user) { @@ -302,7 +303,7 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab return nil, err } - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) m.lock.Lock() defer m.lock.Unlock() diff --git a/pkg/storage/favorite/memory/memory_test.go b/pkg/storage/favorite/memory/memory_test.go index d9869c6d99..7a19b5f401 100644 --- a/pkg/storage/favorite/memory/memory_test.go +++ b/pkg/storage/favorite/memory/memory_test.go @@ -24,7 +24,7 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/appctx" ) type environment struct { @@ -51,11 +51,11 @@ func createEnvironment() environment { return environment{ userOne: userOne, - userOneCtx: ctxpkg.ContextSetUser(context.Background(), userOne), + userOneCtx: appctx.ContextSetUser(context.Background(), userOne), userTwo: userTwo, - userTwoCtx: ctxpkg.ContextSetUser(context.Background(), userTwo), + userTwoCtx: appctx.ContextSetUser(context.Background(), userTwo), userThree: userThree, - userThreeCtx: ctxpkg.ContextSetUser(context.Background(), userThree), + userThreeCtx: appctx.ContextSetUser(context.Background(), userThree), resourceInfoOne: resourceInfoOne, resourceInfoTwo: resourceInfoTwo, diff --git a/pkg/storage/favorite/sql/sql.go b/pkg/storage/favorite/sql/sql.go index 46f8ed98e6..0adfa44a0b 100644 --- a/pkg/storage/favorite/sql/sql.go +++ b/pkg/storage/favorite/sql/sql.go @@ -25,7 +25,8 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/storage/favorite" "github.com/cs3org/reva/pkg/storage/favorite/registry" @@ -68,7 +69,7 @@ func New(m map[string]interface{}) (favorite.Manager, error) { } func (m *mgr) ListFavorites(ctx context.Context, userID *user.UserId) ([]*provider.ResourceId, error) { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) infos := []*provider.ResourceId{} query := `SELECT fileid_prefix, fileid FROM cbox_metadata WHERE uid=? AND tag_key="fav"` rows, err := m.db.Query(query, user.Id.OpaqueId) @@ -92,7 +93,7 @@ func (m *mgr) ListFavorites(ctx context.Context, userID *user.UserId) ([]*provid } func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) // The primary key is just the ID in the table, it should ideally be (uid, fileid_prefix, fileid, tag_key) // For the time being, just check if the favorite already exists. If it does, return early @@ -117,7 +118,7 @@ func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo } func (m *mgr) UnsetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) stmt, err := m.db.Prepare(`DELETE FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"`) if err != nil { return err diff --git a/pkg/storage/fs/cback/cback.go b/pkg/storage/fs/cback/cback.go index 16c608bf8e..0233071839 100644 --- a/pkg/storage/fs/cback/cback.go +++ b/pkg/storage/fs/cback/cback.go @@ -29,7 +29,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" v1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/httpclient" "github.com/cs3org/reva/pkg/mime" @@ -63,7 +64,7 @@ func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err erro func (fs *cback) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { var ssID, searchPath string - user, inContext := ctxpkg.ContextGetUser(ctx) + user, inContext := appctx.ContextGetUser(ctx) if !inContext { return nil, errtypes.UserRequired("user not found in context") @@ -151,7 +152,7 @@ func (fs *cback) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys var path = ref.GetPath() var ssID, searchPath string - user, inContext := ctxpkg.ContextGetUser(ctx) + user, inContext := appctx.ContextGetUser(ctx) if !inContext { return nil, errtypes.UserRequired("no user found in context") @@ -304,7 +305,7 @@ func (fs *cback) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys func (fs *cback) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { var path = ref.GetPath() var ssID, searchPath string - user, _ := ctxpkg.ContextGetUser(ctx) + user, _ := appctx.ContextGetUser(ctx) resp, err := fs.matchBackups(user.Username, path) diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 3382568b2f..e234a17d5e 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -33,7 +33,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" @@ -131,7 +131,7 @@ func NewStorageDriver(c *StorageDriverConfig) (*StorageDriver, error) { } func getUser(ctx context.Context) (*user.User, error) { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { err := errors.Wrap(errtypes.UserRequired(""), "nextcloud storage driver: error getting user from ctx") return nil, err diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index 8be30a838e..8faac01079 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -30,7 +30,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/storage/fs/nextcloud" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" @@ -103,9 +104,9 @@ var _ = Describe("Nextcloud", func() { Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) + ctx = appctx.ContextSetUser(ctx, user) }) AfterEach(func() { diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index 244a1ae699..02d2889a37 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -26,7 +26,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" @@ -87,7 +88,7 @@ type reg struct { func getProviderAddr(ctx context.Context, r rule) string { addr := r.Address if addr == "" { - if u, ok := ctxpkg.ContextGetUser(ctx); ok { + if u, ok := appctx.ContextGetUser(ctx); ok { layout := templates.WithUser(u, r.Mapping) for k, v := range r.Aliases { if match, _ := regexp.MatchString("^"+k, layout); match { diff --git a/pkg/storage/registry/static/static_test.go b/pkg/storage/registry/static/static_test.go index 5476fdea75..6d5ffabeeb 100644 --- a/pkg/storage/registry/static/static_test.go +++ b/pkg/storage/registry/static/static_test.go @@ -24,7 +24,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/storage/registry/static" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -81,12 +82,12 @@ var _ = Describe("Static", func() { }) Expect(err).ToNot(HaveOccurred()) - ctxAlice := ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + ctxAlice := appctx.ContextSetUser(context.Background(), &userpb.User{ Id: &userpb.UserId{ OpaqueId: "alice", }, }) - ctxRobert := ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + ctxRobert := appctx.ContextSetUser(context.Background(), &userpb.User{ Id: &userpb.UserId{ OpaqueId: "robert", }, diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 0f6f4134de..b29ce5c76e 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -43,7 +43,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/eosclient/eosbinary" "github.com/cs3org/reva/pkg/eosclient/eosgrpc" @@ -271,7 +271,7 @@ func (fs *eosfs) Shutdown(ctx context.Context) error { } func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { err := errors.Wrap(errtypes.UserRequired(""), "eosfs: error getting user from ctx") return nil, err @@ -842,7 +842,7 @@ func (fs *eosfs) getUserFromID(ctx context.Context, userID *userpb.UserId) (*use } func (fs *eosfs) userHasWriteAccess(ctx context.Context, user *userpb.User, ref *provider.Reference) (bool, error) { - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetUser(ctx, user) resInfo, err := fs.GetMD(ctx, ref, nil) if err != nil { return false, err @@ -859,7 +859,7 @@ func (fs *eosfs) userIDHasWriteAccess(ctx context.Context, userID *userpb.UserId } func (fs *eosfs) userHasReadAccess(ctx context.Context, user *userpb.User, ref *provider.Reference) (bool, error) { - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetUser(ctx, user) resInfo, err := fs.GetMD(ctx, ref, nil) if err != nil { return false, err @@ -1553,7 +1553,7 @@ func (fs *eosfs) CreateHome(ctx context.Context) error { } func (fs *eosfs) runPostCreateHomeHook(ctx context.Context) error { - user := ctxpkg.ContextMustGetUser(ctx) + user := appctx.ContextMustGetUser(ctx) return exec.Command(fs.conf.OnPostCreateHomeHook, user.Username).Run() } @@ -2091,7 +2091,7 @@ func (fs *eosfs) convertToFileReference(ctx context.Context, eosFileInfo *eoscli // permissionSet returns the permission set for the current user. func (fs *eosfs) permissionSet(ctx context.Context, eosFileInfo *eosclient.FileInfo, owner *userpb.UserId) *provider.ResourcePermissions { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok || u.Id == nil { return &provider.ResourcePermissions{ // no permissions diff --git a/pkg/storage/utils/eosfs/eosfs_test.go b/pkg/storage/utils/eosfs/eosfs_test.go index c2ef3a8947..2f89638d69 100644 --- a/pkg/storage/utils/eosfs/eosfs_test.go +++ b/pkg/storage/utils/eosfs/eosfs_test.go @@ -30,7 +30,8 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" "github.com/gdexlab/go-render/render" "github.com/thanhpk/randstr" @@ -258,7 +259,7 @@ func TestAddGrant(t *testing.T) { for _, test := range testCases { t.Run(test.description, func(t *testing.T) { - ctx := ctxpkg.ContextSetUser(context.TODO(), &userv1beta1.User{ + ctx := appctx.ContextSetUser(context.TODO(), &userv1beta1.User{ UidNumber: 138406, GidNumber: 2763, }) diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index b204b0fe47..fedc2353af 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -35,7 +35,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" @@ -152,7 +153,7 @@ func (fs *localfs) resolve(ctx context.Context, ref *provider.Reference) (p stri } func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { err := errors.Wrap(errtypes.UserRequired(""), "local: error getting user from ctx") return nil, err @@ -272,7 +273,7 @@ func (fs *localfs) isShareFolderChild(ctx context.Context, p string) bool { // permissionSet returns the permission set for the current user. func (fs *localfs) permissionSet(ctx context.Context, owner *userpb.UserId) *provider.ResourcePermissions { - u, ok := ctxpkg.ContextGetUser(ctx) + u, ok := appctx.ContextGetUser(ctx) if !ok { return &provider.ResourcePermissions{ // no permissions diff --git a/pkg/storage/utils/localfs/upload.go b/pkg/storage/utils/localfs/upload.go index 0c2727a537..f4ba06e5e3 100644 --- a/pkg/storage/utils/localfs/upload.go +++ b/pkg/storage/utils/localfs/upload.go @@ -28,7 +28,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/utils" @@ -159,7 +159,7 @@ func (fs *localfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tu if err != nil { return nil, errors.Wrap(err, "localfs: error resolving upload path") } - usr := ctxpkg.ContextMustGetUser(ctx) + usr := appctx.ContextMustGetUser(ctx) info.Storage = map[string]string{ "Type": "LocalStore", "BinPath": binPath, @@ -236,7 +236,7 @@ func (fs *localfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error Username: info.Storage["UserName"], } - ctx = ctxpkg.ContextSetUser(ctx, u) + ctx = appctx.ContextSetUser(ctx, u) return &fileUpload{ info: info, diff --git a/pkg/user/manager/nextcloud/nextcloud_test.go b/pkg/user/manager/nextcloud/nextcloud_test.go index 88fa5ec3a6..b627aa4e17 100644 --- a/pkg/user/manager/nextcloud/nextcloud_test.go +++ b/pkg/user/manager/nextcloud/nextcloud_test.go @@ -23,7 +23,8 @@ import ( "os" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/pkg/user/manager/nextcloud" @@ -100,9 +101,9 @@ var _ = Describe("Nextcloud", func() { Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) + ctx = appctx.ContextSetUser(ctx, user) }) AfterEach(func() { diff --git a/tests/integration/grpc/ocm_invitation_test.go b/tests/integration/grpc/ocm_invitation_test.go index 6228dd82d5..f6da7f0e53 100644 --- a/tests/integration/grpc/ocm_invitation_test.go +++ b/tests/integration/grpc/ocm_invitation_test.go @@ -32,7 +32,8 @@ import ( ocmproviderpb "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" @@ -57,9 +58,9 @@ func ctxWithAuthToken(tokenManager token.Manager, user *userpb.User) context.Con Expect(err).ToNot(HaveOccurred()) tkn, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, tkn) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, tkn) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, tkn) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, tkn) + ctx = appctx.ContextSetUser(ctx, user) return ctx } @@ -307,9 +308,9 @@ var _ = Describe("ocm invitation workflow", func() { cernboxURL = revads["cernboxhttp"].GrpcAddress var ok bool - tknMarie, ok = ctxpkg.ContextGetToken(ctxMarie) + tknMarie, ok = appctx.ContextGetToken(ctxMarie) Expect(ok).To(BeTrue()) - tknEinstein, ok = ctxpkg.ContextGetToken(ctxEinstein) + tknEinstein, ok = appctx.ContextGetToken(ctxEinstein) Expect(ok).To(BeTrue()) tknRes, err := cernboxgw.GenerateInviteToken(ctxEinstein, &invitepb.GenerateInviteTokenRequest{}) diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 6e367f2962..cf08782594 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -23,7 +23,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/appctx" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" @@ -61,9 +62,9 @@ var _ = Describe("user providers", func() { Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user, scope) Expect(err).ToNot(HaveOccurred()) - ctx = ctxpkg.ContextSetToken(ctx, t) - ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, t) - ctx = ctxpkg.ContextSetUser(ctx, user) + ctx = appctx.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, t) + ctx = appctx.ContextSetUser(ctx, user) revads, err = startRevads(dependencies, nil, nil, map[string]string{}) Expect(err).ToNot(HaveOccurred()) From b503ae997663e92c561822ca9777700dc0f4a415 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 9 Oct 2023 20:18:12 +0200 Subject: [PATCH 21/25] update docs --- .../grpc/services/ocminvitemanager/_index.md | 2 +- .../grpc/services/ocmshareprovider/_index.md | 2 +- .../grpc/services/storageprovider/_index.md | 18 +++++++++--------- .../config/http/services/appprovider/_index.md | 2 +- .../config/http/services/archiver/_index.md | 2 +- .../config/http/services/datagateway/_index.md | 2 +- .../config/http/services/ocmprovider/_index.md | 18 +++++++++--------- .../config/http/services/overleaf/_index.md | 8 ++++---- .../http/services/owncloud/ocdav/_index.md | 4 ++-- .../packages/auth/manager/oidc/_index.md | 16 ++++++++-------- .../ocm/provider/authorizer/mentix/_index.md | 2 +- .../packages/user/manager/nextcloud/_index.md | 2 +- 12 files changed, 39 insertions(+), 39 deletions(-) diff --git a/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md b/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md index 871faa3ce8..363d33b540 100644 --- a/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md +++ b/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="provider_domain" type="string" default="The same domain registered in the provider authorizer" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocminvitemanager/ocminvitemanager.go#L62) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocminvitemanager/ocminvitemanager.go#L61) {{< highlight toml >}} [grpc.services.ocminvitemanager] provider_domain = "The same domain registered in the provider authorizer" diff --git a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md index 08235771f3..f68b1ea656 100644 --- a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="provider_domain" type="string" default="The same domain registered in the provider authorizer" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L70) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L71) {{< highlight toml >}} [grpc.services.ocmshareprovider] provider_domain = "The same domain registered in the provider authorizer" diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index ecd6f8eb34..6c1b9fbb75 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mount_path" type="string" default="/" %}} -The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) +The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) {{< highlight toml >}} [grpc.services.storageprovider] mount_path = "/" @@ -17,7 +17,7 @@ mount_path = "/" {{% /dir %}} {{% dir name="mount_id" type="string" default="-" %}} -The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) +The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) {{< highlight toml >}} [grpc.services.storageprovider] mount_id = "-" @@ -25,7 +25,7 @@ mount_id = "-" {{% /dir %}} {{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) {{< highlight toml >}} [grpc.services.storageprovider] driver = "localhome" @@ -33,7 +33,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) {{< highlight toml >}} [grpc.services.storageprovider.drivers.localhome] root = "/var/tmp/reva/" @@ -44,7 +44,7 @@ user_layout = "{{.Username}}" {{% /dir %}} {{% dir name="tmp_folder" type="string" default="/var/tmp" %}} -Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L67) +Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] tmp_folder = "/var/tmp" @@ -52,7 +52,7 @@ tmp_folder = "/var/tmp" {{% /dir %}} {{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L68) +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider] data_server_url = "http://localhost/data" @@ -60,7 +60,7 @@ data_server_url = "http://localhost/data" {{% /dir %}} {{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L69) +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L67) {{< highlight toml >}} [grpc.services.storageprovider] expose_data_server = false @@ -68,7 +68,7 @@ expose_data_server = false {{% /dir %}} {{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L70) +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L68) {{< highlight toml >}} [grpc.services.storageprovider] available_checksums = nil @@ -76,7 +76,7 @@ available_checksums = nil {{% /dir %}} {{% dir name="custom_mime_types_json" type="string" default="nil" %}} -An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L71) +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L69) {{< highlight toml >}} [grpc.services.storageprovider] custom_mime_types_json = "nil" diff --git a/docs/content/en/docs/config/http/services/appprovider/_index.md b/docs/content/en/docs/config/http/services/appprovider/_index.md index bda4901bfc..9c0b7d6059 100644 --- a/docs/content/en/docs/config/http/services/appprovider/_index.md +++ b/docs/content/en/docs/config/http/services/appprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/appprovider/appprovider.go#L56) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/appprovider/appprovider.go#L57) {{< highlight toml >}} [http.services.appprovider] insecure = false diff --git a/docs/content/en/docs/config/http/services/archiver/_index.md b/docs/content/en/docs/config/http/services/archiver/_index.md index 56a4f4f622..391e9a23e0 100644 --- a/docs/content/en/docs/config/http/services/archiver/_index.md +++ b/docs/content/en/docs/config/http/services/archiver/_index.md @@ -9,7 +9,7 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/archiver/handler.go#L63) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/archiver/handler.go#L64) {{< highlight toml >}} [http.services.archiver] insecure = false diff --git a/docs/content/en/docs/config/http/services/datagateway/_index.md b/docs/content/en/docs/config/http/services/datagateway/_index.md index e293489e5d..2e70830e53 100644 --- a/docs/content/en/docs/config/http/services/datagateway/_index.md +++ b/docs/content/en/docs/config/http/services/datagateway/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/datagateway/datagateway.go#L61) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/datagateway/datagateway.go#L62) {{< highlight toml >}} [http.services.datagateway] insecure = false diff --git a/docs/content/en/docs/config/http/services/ocmprovider/_index.md b/docs/content/en/docs/config/http/services/ocmprovider/_index.md index ca19a1691f..0e76159136 100644 --- a/docs/content/en/docs/config/http/services/ocmprovider/_index.md +++ b/docs/content/en/docs/config/http/services/ocmprovider/_index.md @@ -9,23 +9,23 @@ description: > # _struct: config_ {{% dir name="ocm_prefix" type="string" default="ocm" %}} -The prefix URL where the OCM API is served. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L39) +The prefix URL where the OCM API is served. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L40) {{< highlight toml >}} [http.services.ocmprovider] ocm_prefix = "ocm" {{< /highlight >}} {{% /dir %}} -{{% dir name="endpoint" type="string" default="This host's URL. If it's not configured, it is assumed OCM is not available." %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L40) +{{% dir name="endpoint" type="string" default="This host's full URL. If it's not configured, it is assumed OCM is not available." %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L41) {{< highlight toml >}} [http.services.ocmprovider] -endpoint = "This host's URL. If it's not configured, it is assumed OCM is not available." +endpoint = "This host's full URL. If it's not configured, it is assumed OCM is not available." {{< /highlight >}} {{% /dir %}} {{% dir name="provider" type="string" default="reva" %}} -A friendly name that defines this service. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L41) +A friendly name that defines this service. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L42) {{< highlight toml >}} [http.services.ocmprovider] provider = "reva" @@ -33,7 +33,7 @@ provider = "reva" {{% /dir %}} {{% dir name="webdav_root" type="string" default="/remote.php/dav/ocm" %}} -The root URL of the WebDAV endpoint to serve OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L42) +The root URL of the WebDAV endpoint to serve OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L43) {{< highlight toml >}} [http.services.ocmprovider] webdav_root = "/remote.php/dav/ocm" @@ -41,7 +41,7 @@ webdav_root = "/remote.php/dav/ocm" {{% /dir %}} {{% dir name="webapp_root" type="string" default="/external/sciencemesh" %}} -The root URL to serve Web apps via OCM. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L43) +The root URL to serve Web apps via OCM. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L44) {{< highlight toml >}} [http.services.ocmprovider] webapp_root = "/external/sciencemesh" @@ -49,7 +49,7 @@ webapp_root = "/external/sciencemesh" {{% /dir %}} {{% dir name="enable_webapp" type="bool" default=false %}} -Whether web apps are enabled in OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L44) +Whether web apps are enabled in OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L45) {{< highlight toml >}} [http.services.ocmprovider] enable_webapp = false @@ -57,7 +57,7 @@ enable_webapp = false {{% /dir %}} {{% dir name="enable_datatx" type="bool" default=false %}} -Whether data transfers are enabled in OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L45) +Whether data transfers are enabled in OCM shares. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/ocmprovider/ocmprovider.go#L46) {{< highlight toml >}} [http.services.ocmprovider] enable_datatx = false diff --git a/docs/content/en/docs/config/http/services/overleaf/_index.md b/docs/content/en/docs/config/http/services/overleaf/_index.md index fb9b328397..4b44721c5e 100644 --- a/docs/content/en/docs/config/http/services/overleaf/_index.md +++ b/docs/content/en/docs/config/http/services/overleaf/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="app_name" type="string" default="" %}} -The App user-friendly name. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L59) +The App user-friendly name. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L58) {{< highlight toml >}} [http.services.overleaf] app_name = "" @@ -17,7 +17,7 @@ app_name = "" {{% /dir %}} {{% dir name="archiver_url" type="string" default="" %}} -Internet-facing URL of the archiver service, used to serve the files to Overleaf. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L60) +Internet-facing URL of the archiver service, used to serve the files to Overleaf. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L59) {{< highlight toml >}} [http.services.overleaf] archiver_url = "" @@ -25,7 +25,7 @@ archiver_url = "" {{% /dir %}} {{% dir name="app_url" type="string" default="" %}} -The App URL. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L61) +The App URL. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L60) {{< highlight toml >}} [http.services.overleaf] app_url = "" @@ -33,7 +33,7 @@ app_url = "" {{% /dir %}} {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L62) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/overleaf/overleaf.go#L61) {{< highlight toml >}} [http.services.overleaf] insecure = false diff --git a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md index 874587adfb..09f3385228 100644 --- a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md +++ b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md @@ -9,7 +9,7 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L103) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L110) {{< highlight toml >}} [http.services.owncloud.ocdav] insecure = false @@ -17,7 +17,7 @@ insecure = false {{% /dir %}} {{% dir name="notifications" type="map[string]interface{}" default=Settingsg for the Notification Helper %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L114) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L122) {{< highlight toml >}} [http.services.owncloud.ocdav] notifications = Settingsg for the Notification Helper diff --git a/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md index 96d81c6d93..c56eeed2d5 100644 --- a/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md +++ b/docs/content/en/docs/config/packages/auth/manager/oidc/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L63) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L65) {{< highlight toml >}} [auth.manager.oidc] insecure = false @@ -17,7 +17,7 @@ insecure = false {{% /dir %}} {{% dir name="issuer" type="string" default="" %}} -The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L64) +The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L66) {{< highlight toml >}} [auth.manager.oidc] issuer = "" @@ -25,7 +25,7 @@ issuer = "" {{% /dir %}} {{% dir name="id_claim" type="string" default="sub" %}} -The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L65) +The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L67) {{< highlight toml >}} [auth.manager.oidc] id_claim = "sub" @@ -33,7 +33,7 @@ id_claim = "sub" {{% /dir %}} {{% dir name="uid_claim" type="string" default="" %}} -The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L66) +The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L68) {{< highlight toml >}} [auth.manager.oidc] uid_claim = "" @@ -41,7 +41,7 @@ uid_claim = "" {{% /dir %}} {{% dir name="gid_claim" type="string" default="" %}} -The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L67) +The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L69) {{< highlight toml >}} [auth.manager.oidc] gid_claim = "" @@ -49,7 +49,7 @@ gid_claim = "" {{% /dir %}} {{% dir name="gatewaysvc" type="string" default="" %}} -The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L68) +The endpoint at which the GRPC gateway is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L70) {{< highlight toml >}} [auth.manager.oidc] gatewaysvc = "" @@ -57,7 +57,7 @@ gatewaysvc = "" {{% /dir %}} {{% dir name="users_mapping" type="string" default="" %}} - The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L69) + The optional OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L71) {{< highlight toml >}} [auth.manager.oidc] users_mapping = "" @@ -65,7 +65,7 @@ users_mapping = "" {{% /dir %}} {{% dir name="group_claim" type="string" default="" %}} - The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L70) + The group claim to be looked up to map the user (default to 'groups'). [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidc/oidc.go#L72) {{< highlight toml >}} [auth.manager.oidc] group_claim = "" diff --git a/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md b/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md index fe12cab3b7..1194dc447b 100644 --- a/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md +++ b/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/ocm/provider/authorizer/mentix/mentix.go#L79) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/ocm/provider/authorizer/mentix/mentix.go#L81) {{< highlight toml >}} [ocm.provider.authorizer.mentix] insecure = false diff --git a/docs/content/en/docs/config/packages/user/manager/nextcloud/_index.md b/docs/content/en/docs/config/packages/user/manager/nextcloud/_index.md index f24a0f23d0..c4974097ee 100644 --- a/docs/content/en/docs/config/packages/user/manager/nextcloud/_index.md +++ b/docs/content/en/docs/config/packages/user/manager/nextcloud/_index.md @@ -9,7 +9,7 @@ description: > # _struct: UserManagerConfig_ {{% dir name="endpoint" type="string" default="" %}} -The Nextcloud backend endpoint for user management [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/nextcloud/nextcloud.go#L54) +The Nextcloud backend endpoint for user management [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/user/manager/nextcloud/nextcloud.go#L53) {{< highlight toml >}} [user.manager.nextcloud] endpoint = "" From 6318ff2c8bc6c4dcca9a99c35fe91eaf0b4ff568 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 10 Oct 2023 09:46:20 +0200 Subject: [PATCH 22/25] add back ceph example --- examples/ceph/ceph.conf | 3 +++ examples/ceph/keyring | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 examples/ceph/ceph.conf create mode 100644 examples/ceph/keyring diff --git a/examples/ceph/ceph.conf b/examples/ceph/ceph.conf new file mode 100644 index 0000000000..f2daaa8de5 --- /dev/null +++ b/examples/ceph/ceph.conf @@ -0,0 +1,3 @@ +[global] + fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323' + mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]' diff --git a/examples/ceph/keyring b/examples/ceph/keyring new file mode 100644 index 0000000000..9e555cc1b2 --- /dev/null +++ b/examples/ceph/keyring @@ -0,0 +1,2 @@ +[client.admin] + key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA==' From 0d32711a829e669f5e04c7ca6c76e4c5e4c9c491 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 10 Oct 2023 10:17:20 +0200 Subject: [PATCH 23/25] remove otelhttp library --- pkg/rhttp/rhttp.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index 58ad42c62f..b516814a6e 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -32,7 +32,6 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/pkg/errors" "github.com/rs/zerolog" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) type Config func(*Server) @@ -278,7 +277,6 @@ func (s *Server) getHandler() (http.Handler, error) { handler = m(handler) } - handler = otelhttp.NewHandler(handler, "http") return handler, nil } From df017982ce25ad37e7908da5caa40f5c40fb9c26 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 10 Oct 2023 12:09:12 +0200 Subject: [PATCH 24/25] remove files and add tests --- internal/http/interceptors/trace/trace.go | 32 +- .../http/interceptors/trace/trace_test.go | 58 ++ pkg/cbox/share/sql/sql.go | 631 ------------------ .../storage/eoshomewrapper/eoshomewrapper.go | 115 ---- pkg/cbox/storage/eoswrapper/eoswrapper.go | 233 ------- 5 files changed, 79 insertions(+), 990 deletions(-) create mode 100644 internal/http/interceptors/trace/trace_test.go delete mode 100644 pkg/cbox/share/sql/sql.go delete mode 100644 pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go delete mode 100644 pkg/cbox/storage/eoswrapper/eoswrapper.go diff --git a/internal/http/interceptors/trace/trace.go b/internal/http/interceptors/trace/trace.go index cdb5906fcd..710ead452f 100644 --- a/internal/http/interceptors/trace/trace.go +++ b/internal/http/interceptors/trace/trace.go @@ -22,6 +22,7 @@ package trace import ( + "context" "net/http" "github.com/cs3org/reva/pkg/trace" @@ -36,23 +37,32 @@ func New() func(http.Handler) http.Handler { func handler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - // try to get trace from context - traceID := trace.Get(ctx) - if traceID == "" { - // check if traceID is coming from header - traceID = r.Header.Get("X-Trace-ID") - if traceID == "" { - traceID = trace.Generate() - } - ctx = trace.Set(ctx, traceID) - } + traceID, ctx := getTraceID(r) // in case the http service will call a grpc service, // we set the outgoing context so the trace information is // passed through the two protocols. ctx = metadata.AppendToOutgoingContext(ctx, "revad-grpc-trace-id", traceID) + r = r.WithContext(ctx) h.ServeHTTP(w, r) }) } + +func getTraceID(r *http.Request) (string, context.Context) { + ctx := r.Context() + // try to get trace from context + traceID := trace.Get(ctx) + if traceID == "" { + // check if traceID is coming from header + traceID = r.Header.Get("X-Trace-ID") + if traceID == "" { + traceID = r.Header.Get("X-Request-ID") + if traceID == "" { + traceID = trace.Generate() + } + } + ctx = trace.Set(ctx, traceID) + } + return traceID, ctx +} diff --git a/internal/http/interceptors/trace/trace_test.go b/internal/http/interceptors/trace/trace_test.go new file mode 100644 index 0000000000..a956d64de5 --- /dev/null +++ b/internal/http/interceptors/trace/trace_test.go @@ -0,0 +1,58 @@ +package trace + +import ( + "context" + "net/http" + "testing" + + "github.com/cs3org/reva/pkg/trace" +) + +type testPair struct { + e string + r *http.Request +} + +func TestGetTrace(t *testing.T) { + pairs := []*testPair{ + &testPair{ + r: newRequest(context.Background(), map[string]string{"X-Trace-ID": "def"}), + e: "def", + }, + &testPair{ + r: newRequest(context.Background(), map[string]string{"X-Request-ID": "abc"}), + e: "abc", + }, + &testPair{ + r: newRequest(trace.Set(context.Background(), "fgh"), nil), + e: "fgh", + }, + } + + for _, p := range pairs { + got, _ := getTraceID(p.r) + t.Logf("headers: %+v context: %+v got: %+v\n", p.r, p.r.Context(), got) + if got != p.e { + t.Fatal("expected: "+p.e, "got: "+got) + return + } + } + +} + +func newRequest(ctx context.Context, headers map[string]string) *http.Request { + r, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost", nil) + for k, v := range headers { + r.Header.Set(k, v) + } + return r +} + +func TestGenerateTrace(t *testing.T) { + got, _ := getTraceID(newRequest(context.Background(), nil)) + if len(got) != 36 { + t.Fatal("expected random generated UUID 36 chars trace ID but got:" + got) + return + } + +} diff --git a/pkg/cbox/share/sql/sql.go b/pkg/cbox/share/sql/sql.go deleted file mode 100644 index 6352d35eea..0000000000 --- a/pkg/cbox/share/sql/sql.go +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package sql - -import ( - "context" - "database/sql" - "fmt" - "path" - "strconv" - "strings" - "time" - - gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - - "github.com/cs3org/reva/pkg/appctx" - conversions "github.com/cs3org/reva/pkg/cbox/utils" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/share" - "github.com/cs3org/reva/pkg/share/manager/registry" - "github.com/cs3org/reva/pkg/sharedconf" - "github.com/cs3org/reva/pkg/utils" - "github.com/cs3org/reva/pkg/utils/cfg" - - // Provides mysql drivers. - _ "github.com/go-sql-driver/mysql" - "github.com/pkg/errors" - "google.golang.org/genproto/protobuf/field_mask" -) - -const ( - shareTypeUser = 0 - shareTypeGroup = 1 - - projectInstancesPrefix = "newproject" - projectSpaceGroupsPrefix = "cernbox-project-" - projectSpaceAdminGroupsSuffix = "-admins" -) - -func init() { - registry.Register("sql", New) -} - -type config struct { - DBUsername string `mapstructure:"db_username"` - DBPassword string `mapstructure:"db_password"` - DBHost string `mapstructure:"db_host"` - DBPort int `mapstructure:"db_port"` - DBName string `mapstructure:"db_name"` - GatewaySvc string `mapstructure:"gatewaysvc"` -} - -type mgr struct { - c *config - db *sql.DB - client gatewayv1beta1.GatewayAPIClient -} - -func (c *config) ApplyDefaults() { - c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) -} - -// New returns a new share manager. -func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { - var c config - if err := cfg.Decode(m, &c); err != nil { - return nil, err - } - - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DBUsername, c.DBPassword, c.DBHost, c.DBPort, c.DBName)) - if err != nil { - return nil, err - } - - gw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) - if err != nil { - return nil, err - } - - return &mgr{ - c: &c, - db: db, - client: gw, - }, nil -} - -func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { - user := appctx.ContextMustGetUser(ctx) - - // do not allow share to myself or the owner if share is for a user - // TODO(labkode): should not this be caught already at the gw level? - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER && - (utils.UserEqual(g.Grantee.GetUserId(), user.Id) || utils.UserEqual(g.Grantee.GetUserId(), md.Owner)) { - return nil, errors.New("sql: owner/creator and grantee are the same") - } - - // check if share already exists. - key := &collaboration.ShareKey{ - Owner: md.Owner, - ResourceId: md.Id, - Grantee: g.Grantee, - } - _, err := m.getByKey(ctx, key) - - // share already exists - if err == nil { - return nil, errtypes.AlreadyExists(key.String()) - } - - now := time.Now().Unix() - ts := &typespb.Timestamp{ - Seconds: uint64(now), - } - - shareType, shareWith := conversions.FormatGrantee(g.Grantee) - itemType := conversions.ResourceTypeToItem(md.Type) - targetPath := path.Join("/", path.Base(md.Path)) - permissions := conversions.SharePermToInt(g.Permissions.Permissions) - prefix := md.Id.StorageId - itemSource := md.Id.OpaqueId - fileSource, err := strconv.ParseUint(itemSource, 10, 64) - if err != nil { - // it can be the case that the item source may be a character string - // we leave fileSource blank in that case - fileSource = 0 - } - - stmtString := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,share_with=?,file_target=?" - stmtValues := []interface{}{shareType, conversions.FormatUserID(md.Owner), conversions.FormatUserID(user.Id), itemType, prefix, itemSource, fileSource, permissions, now, shareWith, targetPath} - - stmt, err := m.db.Prepare(stmtString) - if err != nil { - return nil, err - } - result, err := stmt.Exec(stmtValues...) - if err != nil { - return nil, err - } - lastID, err := result.LastInsertId() - if err != nil { - return nil, err - } - - return &collaboration.Share{ - Id: &collaboration.ShareId{ - OpaqueId: strconv.FormatInt(lastID, 10), - }, - ResourceId: md.Id, - Permissions: g.Permissions, - Grantee: g.Grantee, - Owner: md.Owner, - Creator: user.Id, - Ctime: ts, - Mtime: ts, - }, nil -} - -func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) { - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - s := conversions.DBShare{ID: id.OpaqueId} - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND id=? AND (uid_owner=? or uid_initiator=?)" - if err := m.db.QueryRow(query, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(id.OpaqueId) - } - return nil, err - } - share := conversions.ConvertToCS3Share(s) - return share, nil -} - -func (m *mgr) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) { - owner := conversions.FormatUserID(key.Owner) - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - - s := conversions.DBShare{} - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, id, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - if err := m.db.QueryRow(query, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(key.String()) - } - return nil, err - } - share := conversions.ConvertToCS3Share(s) - return share, nil -} - -func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { - var s *collaboration.Share - var err error - switch { - case ref.GetId() != nil: - s, err = m.getByID(ctx, ref.GetId()) - case ref.GetKey() != nil: - s, err = m.getByKey(ctx, ref.GetKey()) - default: - err = errtypes.NotFound(ref.String()) - } - - if err != nil { - return nil, err - } - - return s, nil -} - -func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - var query string - params := []interface{}{} - switch { - case ref.GetId() != nil: - query = "delete from oc_share where id=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, ref.GetId().OpaqueId, uid, uid) - case ref.GetKey() != nil: - key := ref.GetKey() - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - owner := conversions.FormatUserID(key.Owner) - query = "delete from oc_share where uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid) - default: - return errtypes.NotFound(ref.String()) - } - - stmt, err := m.db.Prepare(query) - if err != nil { - return err - } - res, err := stmt.Exec(params...) - if err != nil { - return err - } - - rowCnt, err := res.RowsAffected() - if err != nil { - return err - } - if rowCnt == 0 { - return errtypes.NotFound(ref.String()) - } - return nil -} - -func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { - permissions := conversions.SharePermToInt(p.Permissions) - uid := conversions.FormatUserID(appctx.ContextMustGetUser(ctx).Id) - - var query string - params := []interface{}{} - switch { - case ref.GetId() != nil: - query = "update oc_share set permissions=?,stime=? where id=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, permissions, time.Now().Unix(), ref.GetId().OpaqueId, uid, uid) - case ref.GetKey() != nil: - key := ref.GetKey() - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - owner := conversions.FormatUserID(key.Owner) - query = "update oc_share set permissions=?,stime=? where (uid_owner=? or uid_initiator=?) AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)" - params = append(params, permissions, time.Now().Unix(), owner, owner, key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid) - default: - return nil, errtypes.NotFound(ref.String()) - } - - stmt, err := m.db.Prepare(query) - if err != nil { - return nil, err - } - if _, err = stmt.Exec(params...); err != nil { - return nil, err - } - - return m.GetShare(ctx, ref) -} - -func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - id, stime, permissions, share_type - FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (share_type=? OR share_type=?)` - params := []interface{}{shareTypeUser, shareTypeGroup} - - groupedFilters := share.GroupFiltersByType(filters) - if len(groupedFilters) > 0 { - filterQuery, filterParams, err := translateFilters(groupedFilters) - if err != nil { - return nil, err - } - params = append(params, filterParams...) - if filterQuery != "" { - query = fmt.Sprintf("%s AND (%s)", query, filterQuery) - } - } - - uidOwnersQuery, uidOwnersParams, err := m.uidOwnerFilters(ctx, groupedFilters) - if err != nil { - return nil, err - } - params = append(params, uidOwnersParams...) - if uidOwnersQuery != "" { - query = fmt.Sprintf("%s AND (%s)", query, uidOwnersQuery) - } - - rows, err := m.db.Query(query, params...) - if err != nil { - return nil, err - } - defer rows.Close() - - var s conversions.DBShare - shares := []*collaboration.Share{} - for rows.Next() { - if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil { - continue - } - share := conversions.ConvertToCS3Share(s) - shares = append(shares, share) - } - if err = rows.Err(); err != nil { - return nil, err - } - - return shares, nil -} - -// we list the shares that are targeted to the user in context or to the user groups. -func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - params := []interface{}{uid, uid, uid, uid} - for _, v := range user.Groups { - params = append(params, v) - } - - query := `SELECT coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner != ? AND uid_initiator != ?)` - if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" - } else { - query += " AND (share_with=? AND share_type = 0)" - } - - groupedFilters := share.GroupFiltersByType(filters) - filterQuery, filterParams, err := translateFilters(groupedFilters) - if err != nil { - return nil, err - } - params = append(params, filterParams...) - - if filterQuery != "" { - query = fmt.Sprintf("%s AND (%s)", query, filterQuery) - } - - rows, err := m.db.Query(query, params...) - if err != nil { - return nil, err - } - defer rows.Close() - - var s conversions.DBShare - shares := []*collaboration.ReceivedShare{} - for rows.Next() { - if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { - continue - } - share := conversions.ConvertToCS3ReceivedShare(s) - shares = append(shares, share) - } - if err = rows.Err(); err != nil { - return nil, err - } - - return shares, nil -} - -func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - params := []interface{}{uid, id.OpaqueId, uid} - for _, v := range user.Groups { - params = append(params, v) - } - - s := conversions.DBShare{ID: id.OpaqueId} - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND ts.id=?` - if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" - } else { - query += " AND (share_with=? AND share_type = 0)" - } - if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(id.OpaqueId) - } - return nil, err - } - share := conversions.ConvertToCS3ReceivedShare(s) - return share, nil -} - -func (m *mgr) getReceivedByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - shareType, shareWith := conversions.FormatGrantee(key.Grantee) - params := []interface{}{uid, conversions.FormatUserID(key.Owner), key.ResourceId.StorageId, key.ResourceId.OpaqueId, shareType, shareWith, shareWith} - for _, v := range user.Groups { - params = append(params, v) - } - - s := conversions.DBShare{} - query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, - coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, - ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state - FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?) - WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=?` - if len(user.Groups) > 0 { - query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))" - } else { - query += " AND (share_with=? AND share_type = 0)" - } - - if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil { - if err == sql.ErrNoRows { - return nil, errtypes.NotFound(key.String()) - } - return nil, err - } - - share := conversions.ConvertToCS3ReceivedShare(s) - return share, nil -} - -func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { - var s *collaboration.ReceivedShare - var err error - switch { - case ref.GetId() != nil: - s, err = m.getReceivedByID(ctx, ref.GetId()) - case ref.GetKey() != nil: - s, err = m.getReceivedByKey(ctx, ref.GetKey()) - default: - err = errtypes.NotFound(ref.String()) - } - - if err != nil { - return nil, err - } - - return s, nil -} - -func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { - user := appctx.ContextMustGetUser(ctx) - - rs, err := m.GetReceivedShare(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Share.Id}}) - if err != nil { - return nil, err - } - - for i := range fieldMask.Paths { - switch fieldMask.Paths[i] { - case "state": - rs.State = share.State - default: - return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported") - } - } - - state := 0 - switch rs.GetState() { - case collaboration.ShareState_SHARE_STATE_REJECTED: - state = -1 - case collaboration.ShareState_SHARE_STATE_ACCEPTED: - state = 1 - } - - params := []interface{}{rs.Share.Id.OpaqueId, conversions.FormatUserID(user.Id), state, state} - query := "insert into oc_share_status(id, recipient, state) values(?, ?, ?) ON DUPLICATE KEY UPDATE state = ?" - - stmt, err := m.db.Prepare(query) - if err != nil { - return nil, err - } - _, err = stmt.Exec(params...) - if err != nil { - return nil, err - } - - return rs, nil -} - -func (m *mgr) uidOwnerFilters(ctx context.Context, filters map[collaboration.Filter_Type][]*collaboration.Filter) (string, []interface{}, error) { - user := appctx.ContextMustGetUser(ctx) - uid := conversions.FormatUserID(user.Id) - - query := "uid_owner=? or uid_initiator=?" - params := []interface{}{uid, uid} - - client, err := pool.GetGatewayServiceClient(pool.Endpoint(m.c.GatewaySvc)) - if err != nil { - return "", nil, err - } - - if resourceFilters, ok := filters[collaboration.Filter_TYPE_RESOURCE_ID]; ok { - for _, f := range resourceFilters { - // For shares inside project spaces, if the user is an admin, we try to list all shares created by other admins - if strings.HasPrefix(f.GetResourceId().GetStorageId(), projectInstancesPrefix) { - res, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: f.GetResourceId()}}) - if err != nil || res.Status.Code != rpc.Code_CODE_OK { - continue - } - - // The path will look like /eos/project/c/cernbox, we need to extract the project name - parts := strings.SplitN(res.Info.Path, "/", 6) - if len(parts) < 5 { - continue - } - - adminGroup := projectSpaceGroupsPrefix + parts[4] + projectSpaceAdminGroupsSuffix - for _, g := range user.Groups { - if g == adminGroup { - // User belongs to the admin group, list all shares for the resource - - // TODO: this only works if shares for a single project are requested. - // If shares for multiple projects are requested, then we're not checking if the - // user is an admin for all of those. We can append the query ` or uid_owner=?` - // for all the project owners, which works fine for new reva - // but won't work for revaold since there, we store the uid of the share creator as uid_owner. - // For this to work across the two versions, this change would have to be made in revaold - // but it won't be straightforward as there, the storage provider doesn't return the - // resource owners. - return "", []interface{}{}, nil - } - } - } - } - } - - return query, params, nil -} - -func granteeTypeToShareType(granteeType provider.GranteeType) int { - switch granteeType { - case provider.GranteeType_GRANTEE_TYPE_USER: - return shareTypeUser - case provider.GranteeType_GRANTEE_TYPE_GROUP: - return shareTypeGroup - } - return -1 -} - -// translateFilters translates the filters to sql queries. -func translateFilters(filters map[collaboration.Filter_Type][]*collaboration.Filter) (string, []interface{}, error) { - var ( - filterQuery string - params []interface{} - ) - - // If multiple filters of the same type are passed to this function, they need to be combined with the `OR` operator. - // That is why the filters got grouped by type. - // For every given filter type, iterate over the filters and if there are more than one combine them. - // Combine the different filter types using `AND` - var filterCounter = 0 - for filterType, currFilters := range filters { - switch filterType { - case collaboration.Filter_TYPE_RESOURCE_ID: - filterQuery += "(" - for i, f := range currFilters { - filterQuery += "(fileid_prefix =? AND item_source=?)" - params = append(params, f.GetResourceId().StorageId, f.GetResourceId().OpaqueId) - - if i != len(currFilters)-1 { - filterQuery += " OR " - } - } - filterQuery += ")" - case collaboration.Filter_TYPE_GRANTEE_TYPE: - filterQuery += "(" - for i, f := range currFilters { - filterQuery += "share_type=?" - params = append(params, granteeTypeToShareType(f.GetGranteeType())) - - if i != len(currFilters)-1 { - filterQuery += " OR " - } - } - filterQuery += ")" - case collaboration.Filter_TYPE_EXCLUDE_DENIALS: - // TODO this may change once the mapping of permission to share types is completed (cf. pkg/cbox/utils/conversions.go) - filterQuery += "(permissions > 0)" - default: - return "", nil, fmt.Errorf("filter type is not supported") - } - if filterCounter != len(filters)-1 { - filterQuery += " AND " - } - filterCounter++ - } - return filterQuery, params, nil -} diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go deleted file mode 100644 index cee21596a7..0000000000 --- a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package eoshomewrapper - -import ( - "bytes" - "context" - "text/template" - - "github.com/Masterminds/sprig" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/cs3org/reva/pkg/utils/cfg" -) - -func init() { - registry.Register("eoshomewrapper", New) -} - -type wrapper struct { - storage.FS - mountIDTemplate *template.Template -} - -// New returns an implementation of the storage.FS interface that forms a wrapper -// around separate connections to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - var c eosfs.Config - if err := cfg.Decode(m, &c); err != nil { - return nil, err - } - // default to version invariance if not configured - if _, ok := m["version_invariant"]; !ok { - c.VersionInvariant = true - } - - t, ok := m["mount_id_template"].(string) - if !ok || t == "" { - t = "eoshome-{{substr 0 1 .Username}}" - } - - eos, err := eosfs.NewEOSFS(ctx, &c) - if err != nil { - return nil, err - } - - mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t) - if err != nil { - return nil, err - } - - return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil -} - -// We need to override the two methods, GetMD and ListFolder to fill the -// StorageId in the ResourceInfo objects. - -func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { - res, err := w.FS.GetMD(ctx, ref, mdKeys) - if err != nil { - return nil, err - } - - // We need to extract the mount ID based on the mapping template. - // - // Take the first letter of the username of the logged-in user, as the home - // storage provider restricts requests only to the home namespace. - res.Id.StorageId = w.getMountID(ctx, res) - return res, nil -} - -func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { - res, err := w.FS.ListFolder(ctx, ref, mdKeys) - if err != nil { - return nil, err - } - for _, r := range res { - r.Id.StorageId = w.getMountID(ctx, r) - } - return res, nil -} - -func (w *wrapper) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { - return errtypes.NotSupported("eos: deny grant is only enabled for project spaces") -} - -func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string { - u := appctx.ContextMustGetUser(ctx) - b := bytes.Buffer{} - if err := w.mountIDTemplate.Execute(&b, u); err != nil { - return "" - } - return b.String() -} diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go deleted file mode 100644 index 80e5f3a0d0..0000000000 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package eoswrapper - -import ( - "bytes" - "context" - "io" - "strings" - "text/template" - - "github.com/Masterminds/sprig" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/cs3org/reva/pkg/utils/cfg" -) - -func init() { - registry.Register("eoswrapper", New) -} - -const ( - eosProjectsNamespace = "/eos/project" - - // We can use a regex for these, but that might have inferior performance. - projectSpaceGroupsPrefix = "cernbox-project-" - projectSpaceAdminGroupsSuffix = "-admins" -) - -type wrapper struct { - storage.FS - conf *eosfs.Config - mountIDTemplate *template.Template -} - -// New returns an implementation of the storage.FS interface that forms a wrapper -// around separate connections to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - var c eosfs.Config - if err := cfg.Decode(m, &c); err != nil { - return nil, err - } - - // default to version invariance if not configured - if _, ok := m["version_invariant"]; !ok { - c.VersionInvariant = true - } - - // allow recycle operations for project spaces - if !c.EnableHome && strings.HasPrefix(c.Namespace, eosProjectsNamespace) { - c.AllowPathRecycleOperations = true - c.ImpersonateOwnerforRevisions = true - } - - t, ok := m["mount_id_template"].(string) - if !ok || t == "" { - t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}" - } - - eos, err := eosfs.NewEOSFS(ctx, &c) - if err != nil { - return nil, err - } - - mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t) - if err != nil { - return nil, err - } - - return &wrapper{FS: eos, conf: &c, mountIDTemplate: mountIDTemplate}, nil -} - -// We need to override the two methods, GetMD and ListFolder to fill the -// StorageId in the ResourceInfo objects. - -func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { - res, err := w.FS.GetMD(ctx, ref, mdKeys) - if err != nil { - return nil, err - } - - // We need to extract the mount ID based on the mapping template. - // - // Take the first letter of the resource path after the namespace has been removed. - // If it's empty, leave it empty to be filled by storageprovider. - res.Id.StorageId = w.getMountID(ctx, res) - - if err = w.setProjectSharingPermissions(ctx, res); err != nil { - return nil, err - } - - return res, nil -} - -func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { - res, err := w.FS.ListFolder(ctx, ref, mdKeys) - if err != nil { - return nil, err - } - for _, r := range res { - r.Id.StorageId = w.getMountID(ctx, r) - if err = w.setProjectSharingPermissions(ctx, r); err != nil { - continue - } - } - return res, nil -} - -func (w *wrapper) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { - if err := w.userIsProjectAdmin(ctx, ref); err != nil { - return nil, err - } - - return w.FS.ListRevisions(ctx, ref) -} - -func (w *wrapper) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - if err := w.userIsProjectAdmin(ctx, ref); err != nil { - return nil, err - } - - return w.FS.DownloadRevision(ctx, ref, revisionKey) -} - -func (w *wrapper) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { - if err := w.userIsProjectAdmin(ctx, ref); err != nil { - return err - } - - return w.FS.RestoreRevision(ctx, ref, revisionKey) -} - -func (w *wrapper) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { - // This is only allowed for project space admins - if strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) { - if err := w.userIsProjectAdmin(ctx, ref); err != nil { - return err - } - return w.FS.DenyGrant(ctx, ref, g) - } - - return errtypes.NotSupported("eos: deny grant is only enabled for project spaces") -} - -func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string { - if r == nil { - return "" - } - b := bytes.Buffer{} - if err := w.mountIDTemplate.Execute(&b, r); err != nil { - return "" - } - return b.String() -} - -func (w *wrapper) setProjectSharingPermissions(ctx context.Context, r *provider.ResourceInfo) error { - // Check if this storage provider corresponds to a project spaces instance - if strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) { - // Extract project name from the path resembling /c/cernbox or /c/cernbox/minutes/.. - parts := strings.SplitN(r.Path, "/", 4) - if len(parts) != 4 && len(parts) != 3 { - // The request might be for / or /$letter - // Nothing to do in that case - return nil - } - adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix - user := appctx.ContextMustGetUser(ctx) - - for _, g := range user.Groups { - if g == adminGroup { - r.PermissionSet.AddGrant = true - r.PermissionSet.RemoveGrant = true - r.PermissionSet.UpdateGrant = true - r.PermissionSet.ListGrants = true - r.PermissionSet.GetQuota = true - r.PermissionSet.DenyGrant = true - return nil - } - } - } - return nil -} - -func (w *wrapper) userIsProjectAdmin(ctx context.Context, ref *provider.Reference) error { - // Check if this storage provider corresponds to a project spaces instance - if !strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) { - return nil - } - - res, err := w.FS.GetMD(ctx, ref, nil) - if err != nil { - return err - } - - // Extract project name from the path resembling /c/cernbox or /c/cernbox/minutes/.. - parts := strings.SplitN(res.Path, "/", 4) - if len(parts) != 4 && len(parts) != 3 { - // The request might be for / or /$letter - // Nothing to do in that case - return nil - } - adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix - user := appctx.ContextMustGetUser(ctx) - - for _, g := range user.Groups { - if g == adminGroup { - return nil - } - } - - return errtypes.PermissionDenied("eosfs: project spaces revisions can only be accessed by admins") -} From e1605912848e58dbd0492787405fcdd6595ad995 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 10 Oct 2023 14:25:57 +0200 Subject: [PATCH 25/25] align with upstream --- .../packages/storage/registry/_index.md | 7 +++++ .../storage/registry/dynamic/_index.md | 26 +++++++++++++++++++ .../http/interceptors/trace/trace_test.go | 26 +++++++++++++++---- pkg/storage/registry/dynamic/dynamic.go | 4 +-- pkg/storage/registry/dynamic/dynamic_test.go | 8 +++--- .../registry/dynamic/rewriter/userrewriter.go | 4 +-- 6 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 docs/content/en/docs/config/packages/storage/registry/_index.md create mode 100644 docs/content/en/docs/config/packages/storage/registry/dynamic/_index.md diff --git a/docs/content/en/docs/config/packages/storage/registry/_index.md b/docs/content/en/docs/config/packages/storage/registry/_index.md new file mode 100644 index 0000000000..0fac6a5ca7 --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/registry/_index.md @@ -0,0 +1,7 @@ +--- +title: "registry" +linkTitle: "registry" +weight: 10 +description: > + Configuration for the registry service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/storage/registry/dynamic/_index.md b/docs/content/en/docs/config/packages/storage/registry/dynamic/_index.md new file mode 100644 index 0000000000..f7fda188cc --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/registry/dynamic/_index.md @@ -0,0 +1,26 @@ +--- +title: "dynamic" +linkTitle: "dynamic" +weight: 10 +description: > + Configuration for the dynamic service +--- + +# _struct: config_ + +{{% dir name="rules" type="map[string]string" default=nil %}} +A map from mountID to provider address [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/registry/dynamic/dynamic.go#L53) +{{< highlight toml >}} +[storage.registry.dynamic] +rules = nil +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="rewrites" type="map[string]string" default=nil %}} +A map from a path to an template alias to use when resolving [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/registry/dynamic/dynamic.go#L54) +{{< highlight toml >}} +[storage.registry.dynamic] +rewrites = nil +{{< /highlight >}} +{{% /dir %}} + diff --git a/internal/http/interceptors/trace/trace_test.go b/internal/http/interceptors/trace/trace_test.go index a956d64de5..3f54f0ef98 100644 --- a/internal/http/interceptors/trace/trace_test.go +++ b/internal/http/interceptors/trace/trace_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package trace import ( @@ -15,15 +33,15 @@ type testPair struct { func TestGetTrace(t *testing.T) { pairs := []*testPair{ - &testPair{ + { r: newRequest(context.Background(), map[string]string{"X-Trace-ID": "def"}), e: "def", }, - &testPair{ + { r: newRequest(context.Background(), map[string]string{"X-Request-ID": "abc"}), e: "abc", }, - &testPair{ + { r: newRequest(trace.Set(context.Background(), "fgh"), nil), e: "fgh", }, @@ -37,7 +55,6 @@ func TestGetTrace(t *testing.T) { return } } - } func newRequest(ctx context.Context, headers map[string]string) *http.Request { @@ -54,5 +71,4 @@ func TestGenerateTrace(t *testing.T) { t.Fatal("expected random generated UUID 36 chars trace ID but got:" + got) return } - } diff --git a/pkg/storage/registry/dynamic/dynamic.go b/pkg/storage/registry/dynamic/dynamic.go index 94602e5411..32053250a4 100644 --- a/pkg/storage/registry/dynamic/dynamic.go +++ b/pkg/storage/registry/dynamic/dynamic.go @@ -50,8 +50,8 @@ type dynamic struct { } type config struct { - Rules map[string]string `mapstructure:"rules" docs:"nil;A map from mountID to provider address"` - Rewrites map[string]string `mapstructure:"rewrites" docs:"nil;A map from a path to an template alias to use when resolving"` + Rules map[string]string `docs:"nil;A map from mountID to provider address" mapstructure:"rules"` + Rewrites map[string]string `docs:"nil;A map from a path to an template alias to use when resolving" mapstructure:"rewrites"` HomePath string `mapstructure:"home_path"` DBUsername string `mapstructure:"db_username"` DBPassword string `mapstructure:"db_password"` diff --git a/pkg/storage/registry/dynamic/dynamic_test.go b/pkg/storage/registry/dynamic/dynamic_test.go index 3444441222..0336481d41 100644 --- a/pkg/storage/registry/dynamic/dynamic_test.go +++ b/pkg/storage/registry/dynamic/dynamic_test.go @@ -26,7 +26,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registryv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" @@ -53,17 +53,17 @@ var _ = Describe("Dynamic storage provider", func() { s *server.Server m sync.Mutex - ctxAlice = ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + ctxAlice = appctx.ContextSetUser(context.Background(), &userpb.User{ Id: &userpb.UserId{ OpaqueId: "alice", }, }) - ctxBob = ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + ctxBob = appctx.ContextSetUser(context.Background(), &userpb.User{ Id: &userpb.UserId{ OpaqueId: "bob", }, }) - ctxCharlie = ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + ctxCharlie = appctx.ContextSetUser(context.Background(), &userpb.User{ Id: &userpb.UserId{ OpaqueId: "charlie", }, diff --git a/pkg/storage/registry/dynamic/rewriter/userrewriter.go b/pkg/storage/registry/dynamic/rewriter/userrewriter.go index 1be412051e..5e21ec78ae 100644 --- a/pkg/storage/registry/dynamic/rewriter/userrewriter.go +++ b/pkg/storage/registry/dynamic/rewriter/userrewriter.go @@ -24,7 +24,7 @@ import ( "errors" "strings" - ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/storage/utils/templates" ) @@ -51,7 +51,7 @@ func (ur UserRewriter) GetAlias(ctx context.Context, route string) string { return route } - if u, ok := ctxpkg.ContextGetUser(ctx); ok { + if u, ok := appctx.ContextGetUser(ctx); ok { return templates.WithUser(u, tpl) }