Skip to content

Commit

Permalink
Merge pull request #27 from stv0g/fix-26
Browse files Browse the repository at this point in the history 
Allow fine-grained configuration of S3 server setup
  • Loading branch information
@stv0g
stv0g authored Jun 12, 2022
2 parents 8d5c2f1 + b4d1d5f commit a7b26db
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 70 deletions.
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,18 @@ All settings from the configuration file can also be set via environment variabl
| :-- | :-- | :-- |
| `GOSE_LISTEN` | `":8080"` | Listen address and port of Gose |
| `GOSE_BASE_URL` | `"http://localhost:8080"` | Base URL at which Gose is accessible |
| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets if not bundled into the binary |
| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets (pre-compiled binaries of GoSƐ come with assets embedded into binary.) |
| `GOSE_BUCKET` | `gose-uploads` | Name of S3 bucket |
| `GOSE_ENDPOINT` | (without `http(s)://` prefix, but with port number) | Hostname:Port of S3 server |
| `GOSE_REGION` | `us-east-1` | Region of S3 server |
| `GOSE_PATH_STYLE` | `false` | Prepend bucket name to path |
| `GOSE_NO_SSL` | `false` | Disable SSL encryption for S3 |
| `GOSE_ACCESS_KEY` | | S3 Access Key |
| `GOSE_SECRET_KEY` | | S3 Secret Key |
| `GOSE_CREATE_BUCKET` | `true` | Create S3 bucket if non-existant |
| `GOSE_SETUP_BUCKET` | `true` | Create S3 bucket if do not exists |
| `GOSE_SETUP_CORS` | `true` (if supported by S3 implementation) | Setup S3 bucket CORS rules |
| `GOSE_SETUP_LIFECYCLE` | `true` (if supported by S3 implementation) | Setup S3 bucket lifecycle rules |
| `GOSE_SETUP_ABORT_INCOMPLETE_UPLOADS` | `31` | Number of days after which incomplete uploads are cleaned-up (set to 0 to disable) |
| `GOSE_MAX_UPLOAD_SIZE` | `1TB` | Maximum upload size |
| `GOSE_PART_SIZE` | `16MB` | Part-size for multi-part uploads |
| `AWS_ACCESS_KEY_ID` | | alias for `GOSE_ACCESS_KEY` |
Expand Down
22 changes: 19 additions & 3 deletions config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,28 @@ servers:
access_key: ""
secret_key: ""

# Create the bucket if it does not exist
create_bucket: true

max_upload_size: 5TB
part_size: 16MB

# Manual configuration of S3 implementation
# Usually its auto-detected so the is only required in case
# a proxy or CDN manipulates the "Server" HTTP-response header
# implementation: MinIO

setup:
# Create the bucket if it does not exist
bucket: true

# Setup CORS rules for S3 bucket
cors: true

# Setup lifecycle rules for object expiration
# The rules are defined by the following expiration setting
lifecycle: true

# Number of days after which incomplete uploads are cleaned-up (set to 0 to disable)
abort_incomplete_uploads: 31

# A list of expiration/rentention classes
# The first class is selected by default
expiration:
Expand Down
37 changes: 26 additions & 11 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,34 @@ type S3ServerConfig struct {
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`

MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"`
PartSize size `json:"part_size" yaml:"part_size"`
Expiration []Expiration `json:"expiration" yaml:"expiration"`
Implementation string `json:"implementation" yaml:"implementation"`
MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"`
PartSize size `json:"part_size" yaml:"part_size"`
Expiration []Expiration `json:"expiration" yaml:"expiration"`
}

// S3ServerSetup describes initial configuration for an S3 server/bucket
type S3ServerSetup struct {
CreateBucket bool `json:"create_bucket" yaml:"create_bucket"`
CORS bool `json:"cors" yaml:"cors"`
Lifecycle bool `json:"lifecycle" yaml:"ifecycle"`
AbortIncompleteUploads int `json:"abort_incomplete_uploads" yaml:"abort_incomplete_uploads"`
}

// S3Server describes an S3 server
type S3Server struct {
// S3ServerConfig is the public info about an S3 server shared with the frontend
S3ServerConfig `json:",squash"`

Endpoint string `json:"endpoint" yaml:"endpoint"`
Bucket string `json:"bucket" yaml:"bucket"`
Region string `json:"region" yaml:"region"`
PathStyle bool `json:"path_style" yaml:"path_style"`
NoSSL bool `json:"no_ssl" yaml:"no_ssl"`
AccessKey string `json:"access_key" yaml:"access_key"`
SecretKey string `json:"secret_key" yaml:"secret_key"`
CreateBucket bool `json:"create_bucket" yaml:"create_bucket"`
Endpoint string `json:"endpoint" yaml:"endpoint"`
Bucket string `json:"bucket" yaml:"bucket"`
Region string `json:"region" yaml:"region"`
PathStyle bool `json:"path_style" yaml:"path_style"`
NoSSL bool `json:"no_ssl" yaml:"no_ssl"`
AccessKey string `json:"access_key" yaml:"access_key"`
SecretKey string `json:"secret_key" yaml:"secret_key"`

Setup S3ServerSetup `json:"setup" yaml:"setup"`
}

// ShortenerConfig contains Link-shortener specific configuration
Expand Down Expand Up @@ -167,6 +178,10 @@ func NewConfig(configFile string) (*Config, error) {
cfg.SetDefault("access_key", "")
cfg.SetDefault("secret_key", "")
cfg.SetDefault("create_bucket", true)
cfg.SetDefault("implementation", "")
cfg.SetDefault("setup.cors", true)
cfg.SetDefault("setup.lifecycle", true)
cfg.SetDefault("setup.abort_incomplete_uploads", 31)

cfg.BindEnv("access_key", "AWS_ACCESS_KEY_ID")
cfg.BindEnv("secret_key", "AWS_SECRET_ACCESS_KEY")
Expand Down
8 changes: 2 additions & 6 deletions pkg/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import (
"github.com/stv0g/gose/pkg/config"
)

type Implementation string

const (
ImplementationAWS = "AmazonS3"
ImplementationMinio = "MinIO"
Expand All @@ -25,8 +23,6 @@ type Server struct {
*s3.S3

Config *config.S3Server

Implementation Implementation
}

// GetURL returns the full endpoint URL of the S3 server
Expand Down Expand Up @@ -69,7 +65,7 @@ func (s *Server) GetExpirationClass(cls string) *config.Expiration {
return nil
}

func (s *Server) DetectImplementation() Implementation {
func (s *Server) DetectImplementation() string {
if strings.Contains(s.Config.Endpoint, "digitaloceanspaces.com") {
return ImplementationDigitalOceanSpaces
} else if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") {
Expand All @@ -83,7 +79,7 @@ func (s *Server) DetectImplementation() Implementation {
}
if err := req.Send(); err == nil {
if svr := req.HTTPResponse.Header.Get("Server"); svr != "" {
return Implementation(svr)
return svr
}

return ImplementationUnknown
Expand Down
100 changes: 52 additions & 48 deletions pkg/server/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,24 @@ import (

// Setup initializes the S3 bucket (life-cycle rules & CORS)
func (s *Server) Setup() error {
s.Implementation = s.DetectImplementation()
log.Printf("Detected %s S3 implementation for server %s\n", s.Implementation, s.GetURL())
if s.Config.Implementation == "" {
s.Config.Implementation = s.DetectImplementation()
log.Printf("Detected %s S3 implementation for server %s", s.Config.Implementation, s.GetURL())
} else {
log.Printf("Using %s S3 implementation for server %s", s.Config.Implementation, s.GetURL())
}

// MinIO does not support the setup of bucket CORS rules and MPU abortion lifecycle
if s.Config.Implementation == ImplementationMinio {
s.Config.Setup.CORS = false
s.Config.Setup.AbortIncompleteUploads = 0
}

// Create bucket if it does not exist yet
if _, err := s.GetBucketPolicy(&s3.GetBucketPolicyInput{
Bucket: aws.String(s.Config.Bucket),
}); err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.CreateBucket {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.Setup.CreateBucket {
if _, err := s.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(s.Config.Bucket),
}); err != nil {
Expand All @@ -28,7 +38,7 @@ func (s *Server) Setup() error {
}

// Set CORS configuration for bucket
if s.Implementation != ImplementationMinio {
if s.Config.Setup.CORS {
corsRule := &s3.CORSRule{
AllowedHeaders: aws.StringSlice([]string{"Authorization"}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
Expand All @@ -49,56 +59,50 @@ func (s *Server) Setup() error {
}
}

// Create lifecycle policies
lcRules := []*s3.LifecycleRule{}
if s.Config.Setup.Lifecycle {
// Create lifecycle policies
lcRules := []*s3.LifecycleRule{}

if s.Implementation != ImplementationMinio {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String("Abort Multipart Uploads"),
Status: aws.String("Enabled"),
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
DaysAfterInitiation: aws.Int64(31),
},
Filter: &s3.LifecycleRuleFilter{
Prefix: aws.String("/"),
},
})
}
if s.Config.Setup.AbortIncompleteUploads > 0 {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String("Abort Multipart Uploads"),
Status: aws.String("Enabled"),
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
DaysAfterInitiation: aws.Int64(31),
},
Filter: &s3.LifecycleRuleFilter{
Prefix: aws.String("/"),
},
})
}

for _, cls := range s.Config.Expiration {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{
Tag: &s3.Tag{
Key: aws.String("expiration"),
Value: aws.String(cls.ID),
for _, cls := range s.Config.Expiration {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{
Tag: &s3.Tag{
Key: aws.String("expiration"),
Value: aws.String(cls.ID),
},
},
},
Expiration: &s3.LifecycleExpiration{
Days: aws.Int64(cls.Days),
},
})
}
Expiration: &s3.LifecycleExpiration{
Days: aws.Int64(cls.Days),
},
})
}

if len(lcRules) > 0 {
if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(s.Config.Bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: lcRules,
},
}); err != nil {
return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err)
if len(lcRules) > 0 {
if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(s.Config.Bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: lcRules,
},
}); err != nil {
return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err)
}
}
}

// lc, err := svc.GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{
// Bucket: aws.String(.Bucket),
// })
// if err != nil {
// return fmt.Errorf("failed get life-cycle rules: %w", err)
// }
// log.Printf("Life-cycle rules: %+#v\n", lc)

return nil
}

0 comments on commit a7b26db

Please sign in to comment.