-
Notifications
You must be signed in to change notification settings - Fork 113
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
30 changed files
with
5,243 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| Enhancement: Add s3ng storage driver, storing blobs in a s3-compatible blobstore | ||
|
|
||
| We added a new storage driver (s3ng) which stores the file metadata on a local | ||
| filesystem (reusing the decomposed filesystem of the ocis driver) and the | ||
| actual content as blobs in any s3-compatible blobstore. | ||
|
|
||
| https://github.com/cs3org/reva/pull/1429 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // Copyright 2018-2021 CERN | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| // | ||
| // In applying this license, CERN does not waive the privileges and immunities | ||
| // granted to it by virtue of its status as an Intergovernmental Organization | ||
| // or submit itself to any jurisdiction. | ||
|
|
||
| package blobstore | ||
|
|
||
| import ( | ||
| "io" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/credentials" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| "github.com/aws/aws-sdk-go/service/s3/s3manager" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| // Blobstore provides an interface to an s3 compatible blobstore | ||
| type Blobstore struct { | ||
| s3 *s3.S3 | ||
| uploader *s3manager.Uploader | ||
|
|
||
| bucket string | ||
| } | ||
|
|
||
| // New returns a new Blobstore | ||
| func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, error) { | ||
| sess, err := session.NewSession(&aws.Config{ | ||
| Endpoint: aws.String(endpoint), | ||
| Region: aws.String(region), | ||
| Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), | ||
| S3ForcePathStyle: aws.Bool(true), | ||
| }) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "failed to setup s3 session") | ||
| } | ||
| uploader := s3manager.NewUploader(sess) | ||
|
|
||
| return &Blobstore{ | ||
| uploader: uploader, | ||
| s3: s3.New(sess), | ||
| bucket: bucket, | ||
| }, nil | ||
| } | ||
|
|
||
| // Upload stores some data in the blobstore under the given key | ||
| func (bs *Blobstore) Upload(key string, reader io.Reader) error { | ||
| _, err := bs.uploader.Upload(&s3manager.UploadInput{ | ||
| Bucket: aws.String(bs.bucket), | ||
| Key: aws.String(key), | ||
| Body: reader, | ||
| }) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", key, bs.bucket) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Download retrieves a blob from the blobstore for reading | ||
| func (bs *Blobstore) Download(key string) (io.ReadCloser, error) { | ||
| input := &s3.GetObjectInput{ | ||
| Bucket: aws.String(bs.bucket), | ||
| Key: aws.String(key), | ||
| } | ||
| result, err := bs.s3.GetObject(input) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", key, bs.bucket) | ||
| } | ||
| return result.Body, nil | ||
| } | ||
|
|
||
| // Delete deletes a blob from the blobstore | ||
| func (bs *Blobstore) Delete(key string) error { | ||
| input := &s3.DeleteObjectInput{ | ||
| Bucket: aws.String(bs.bucket), | ||
| Key: aws.String(key), | ||
| } | ||
| _, err := bs.s3.DeleteObject(input) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", key, bs.bucket) | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,169 @@ | ||
| // Copyright 2018-2021 CERN | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| // | ||
| // In applying this license, CERN does not waive the privileges and immunities | ||
| // granted to it by virtue of its status as an Intergovernmental Organization | ||
| // or submit itself to any jurisdiction. | ||
|
|
||
| package s3ng | ||
|
|
||
| import ( | ||
| "context" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" | ||
| "github.com/cs3org/reva/pkg/appctx" | ||
| "github.com/cs3org/reva/pkg/errtypes" | ||
| "github.com/cs3org/reva/pkg/storage/fs/s3ng/node" | ||
| "github.com/cs3org/reva/pkg/storage/fs/s3ng/xattrs" | ||
| "github.com/cs3org/reva/pkg/storage/utils/ace" | ||
| "github.com/pkg/xattr" | ||
| ) | ||
|
|
||
| func (fs *s3ngfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { | ||
| log := appctx.GetLogger(ctx) | ||
| log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") | ||
| var node *node.Node | ||
| if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { | ||
| return | ||
| } | ||
| if !node.Exists { | ||
| err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) | ||
| return | ||
| } | ||
|
|
||
| ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { | ||
| // TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92 | ||
| return rp.AddGrant || rp.UpdateGrant | ||
| }) | ||
| switch { | ||
| case err != nil: | ||
| return errtypes.InternalError(err.Error()) | ||
| case !ok: | ||
| return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) | ||
| } | ||
|
|
||
| np := fs.lu.InternalPath(node.ID) | ||
| e := ace.FromGrant(g) | ||
| principal, value := e.Marshal() | ||
| if err := xattr.Set(np, xattrs.GrantPrefix+principal, value); err != nil { | ||
| return err | ||
| } | ||
| return fs.tp.Propagate(ctx, node) | ||
| } | ||
|
|
||
| func (fs *s3ngfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { | ||
| var node *node.Node | ||
| if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { | ||
| return | ||
| } | ||
| if !node.Exists { | ||
| err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) | ||
| return | ||
| } | ||
|
|
||
| ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { | ||
| return rp.ListGrants | ||
| }) | ||
| switch { | ||
| case err != nil: | ||
| return nil, errtypes.InternalError(err.Error()) | ||
| case !ok: | ||
| return nil, errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) | ||
| } | ||
|
|
||
| log := appctx.GetLogger(ctx) | ||
| np := fs.lu.InternalPath(node.ID) | ||
| var attrs []string | ||
| if attrs, err = xattr.List(np); err != nil { | ||
| log.Error().Err(err).Msg("error listing attributes") | ||
| return nil, err | ||
| } | ||
|
|
||
| log.Debug().Interface("attrs", attrs).Msg("read attributes") | ||
|
|
||
| aces := extractACEsFromAttrs(ctx, np, attrs) | ||
|
|
||
| grants = make([]*provider.Grant, 0, len(aces)) | ||
| for i := range aces { | ||
| grants = append(grants, aces[i].Grant()) | ||
| } | ||
|
|
||
| return grants, nil | ||
| } | ||
|
|
||
| func (fs *s3ngfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { | ||
| var node *node.Node | ||
| if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { | ||
| return | ||
| } | ||
| if !node.Exists { | ||
| err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) | ||
| return | ||
| } | ||
|
|
||
| ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { | ||
| return rp.RemoveGrant | ||
| }) | ||
| switch { | ||
| case err != nil: | ||
| return errtypes.InternalError(err.Error()) | ||
| case !ok: | ||
| return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) | ||
| } | ||
|
|
||
| var attr string | ||
| if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { | ||
| attr = xattrs.GrantPrefix + xattrs.GroupAcePrefix + g.Grantee.Id.OpaqueId | ||
| } else { | ||
| attr = xattrs.GrantPrefix + xattrs.UserAcePrefix + g.Grantee.Id.OpaqueId | ||
| } | ||
|
|
||
| np := fs.lu.InternalPath(node.ID) | ||
| if err = xattr.Remove(np, attr); err != nil { | ||
| return | ||
| } | ||
|
|
||
| return fs.tp.Propagate(ctx, node) | ||
| } | ||
|
|
||
| func (fs *s3ngfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { | ||
| // TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92 | ||
| return fs.AddGrant(ctx, ref, g) | ||
| } | ||
|
|
||
| // extractACEsFromAttrs reads ACEs in the list of attrs from the node | ||
| func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { | ||
| log := appctx.GetLogger(ctx) | ||
| entries = []*ace.ACE{} | ||
| for i := range attrs { | ||
| if strings.HasPrefix(attrs[i], xattrs.GrantPrefix) { | ||
| var value []byte | ||
| var err error | ||
| if value, err = xattr.Get(fsfn, attrs[i]); err != nil { | ||
| log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") | ||
| continue | ||
| } | ||
| var e *ace.ACE | ||
| principal := attrs[i][len(xattrs.GrantPrefix):] | ||
| if e, err = ace.Unmarshal(principal, value); err != nil { | ||
| log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace") | ||
| continue | ||
| } | ||
| entries = append(entries, e) | ||
| } | ||
| } | ||
| return | ||
| } |
Oops, something went wrong.