diff --git a/README.md b/README.md index 13a9ea9..ed2345d 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ All settings from the configuration file can also be set via environment variabl | :-- | :-- | :-- | | `GOSE_LISTEN` | `":8080"` | Listen address and port of Gose | | `GOSE_BASE_URL` | `"http://localhost:8080"` | Base URL at which Gose is accessible | -| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets if not bundled into the binary | +| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets (pre-compiled binaries of GoSƐ come with assets embedded into binary.) | | `GOSE_BUCKET` | `gose-uploads` | Name of S3 bucket | | `GOSE_ENDPOINT` | (without `http(s)://` prefix, but with port number) | Hostname:Port of S3 server | | `GOSE_REGION` | `us-east-1` | Region of S3 server | @@ -123,7 +123,10 @@ All settings from the configuration file can also be set via environment variabl | `GOSE_NO_SSL` | `false` | Disable SSL encryption for S3 | | `GOSE_ACCESS_KEY` | | S3 Access Key | | `GOSE_SECRET_KEY` | | S3 Secret Key | -| `GOSE_CREATE_BUCKET` | `true` | Create S3 bucket if non-existant | +| `GOSE_SETUP_BUCKET` | `true` | Create S3 bucket if do not exists | +| `GOSE_SETUP_CORS` | `true` (if supported by S3 implementation) | Setup S3 bucket CORS rules | +| `GOSE_SETUP_LIFECYCLE` | `true` (if supported by S3 implementation) | Setup S3 bucket lifecycle rules | +| `GOSE_SETUP_ABORT_INCOMPLETE_UPLOADS` | `31` | Number of days after which incomplete uploads are cleaned-up (set to 0 to disable) | | `GOSE_MAX_UPLOAD_SIZE` | `1TB` | Maximum upload size | | `GOSE_PART_SIZE` | `16MB` | Part-size for multi-part uploads | | `AWS_ACCESS_KEY_ID` | | alias for `GOSE_ACCESS_KEY` | diff --git a/config.yaml b/config.yaml index b3cf27f..9e7d3ed 100644 --- a/config.yaml +++ b/config.yaml @@ -26,12 +26,28 @@ servers: access_key: "" secret_key: "" - # Create the bucket if it does not exist - create_bucket: true - max_upload_size: 5TB part_size: 16MB + # Manual configuration of S3 implementation + # Usually its auto-detected so the is only required in case + # a proxy or CDN manipulates the "Server" HTTP-response header + # implementation: MinIO + + setup: + # Create the bucket if it does not exist + bucket: true + + # Setup CORS rules for S3 bucket + cors: true + + # Setup lifecycle rules for object expiration + # The rules are defined by the following expiration setting + lifecycle: true + + # Number of days after which incomplete uploads are cleaned-up (set to 0 to disable) + abort_incomplete_uploads: 31 + # A list of expiration/rentention classes # The first class is selected by default expiration: diff --git a/pkg/config/config.go b/pkg/config/config.go index 1c49ddf..aa269fe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -81,23 +81,34 @@ type S3ServerConfig struct { ID string `json:"id" yaml:"id"` Title string `json:"title" yaml:"title"` - MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"` - PartSize size `json:"part_size" yaml:"part_size"` - Expiration []Expiration `json:"expiration" yaml:"expiration"` + Implementation string `json:"implementation" yaml:"implementation"` + MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"` + PartSize size `json:"part_size" yaml:"part_size"` + Expiration []Expiration `json:"expiration" yaml:"expiration"` +} + +// S3ServerSetup describes initial configuration for an S3 server/bucket +type S3ServerSetup struct { + CreateBucket bool `json:"create_bucket" yaml:"create_bucket"` + CORS bool `json:"cors" yaml:"cors"` + Lifecycle bool `json:"lifecycle" yaml:"ifecycle"` + AbortIncompleteUploads int `json:"abort_incomplete_uploads" yaml:"abort_incomplete_uploads"` } // S3Server describes an S3 server type S3Server struct { + // S3ServerConfig is the public info about an S3 server shared with the frontend S3ServerConfig `json:",squash"` - Endpoint string `json:"endpoint" yaml:"endpoint"` - Bucket string `json:"bucket" yaml:"bucket"` - Region string `json:"region" yaml:"region"` - PathStyle bool `json:"path_style" yaml:"path_style"` - NoSSL bool `json:"no_ssl" yaml:"no_ssl"` - AccessKey string `json:"access_key" yaml:"access_key"` - SecretKey string `json:"secret_key" yaml:"secret_key"` - CreateBucket bool `json:"create_bucket" yaml:"create_bucket"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + Bucket string `json:"bucket" yaml:"bucket"` + Region string `json:"region" yaml:"region"` + PathStyle bool `json:"path_style" yaml:"path_style"` + NoSSL bool `json:"no_ssl" yaml:"no_ssl"` + AccessKey string `json:"access_key" yaml:"access_key"` + SecretKey string `json:"secret_key" yaml:"secret_key"` + + Setup S3ServerSetup `json:"setup" yaml:"setup"` } // ShortenerConfig contains Link-shortener specific configuration @@ -167,6 +178,10 @@ func NewConfig(configFile string) (*Config, error) { cfg.SetDefault("access_key", "") cfg.SetDefault("secret_key", "") cfg.SetDefault("create_bucket", true) + cfg.SetDefault("implementation", "") + cfg.SetDefault("setup.cors", true) + cfg.SetDefault("setup.lifecycle", true) + cfg.SetDefault("setup.abort_incomplete_uploads", 31) cfg.BindEnv("access_key", "AWS_ACCESS_KEY_ID") cfg.BindEnv("secret_key", "AWS_SECRET_ACCESS_KEY") diff --git a/pkg/server/server.go b/pkg/server/server.go index 0c3792b..76d1daa 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,8 +10,6 @@ import ( "github.com/stv0g/gose/pkg/config" ) -type Implementation string - const ( ImplementationAWS = "AmazonS3" ImplementationMinio = "MinIO" @@ -25,8 +23,6 @@ type Server struct { *s3.S3 Config *config.S3Server - - Implementation Implementation } // GetURL returns the full endpoint URL of the S3 server @@ -69,7 +65,7 @@ func (s *Server) GetExpirationClass(cls string) *config.Expiration { return nil } -func (s *Server) DetectImplementation() Implementation { +func (s *Server) DetectImplementation() string { if strings.Contains(s.Config.Endpoint, "digitaloceanspaces.com") { return ImplementationDigitalOceanSpaces } else if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") { @@ -83,7 +79,7 @@ func (s *Server) DetectImplementation() Implementation { } if err := req.Send(); err == nil { if svr := req.HTTPResponse.Header.Get("Server"); svr != "" { - return Implementation(svr) + return svr } return ImplementationUnknown diff --git a/pkg/server/setup.go b/pkg/server/setup.go index ab6ec85..672a39f 100644 --- a/pkg/server/setup.go +++ b/pkg/server/setup.go @@ -11,14 +11,24 @@ import ( // Setup initializes the S3 bucket (life-cycle rules & CORS) func (s *Server) Setup() error { - s.Implementation = s.DetectImplementation() - log.Printf("Detected %s S3 implementation for server %s\n", s.Implementation, s.GetURL()) + if s.Config.Implementation == "" { + s.Config.Implementation = s.DetectImplementation() + log.Printf("Detected %s S3 implementation for server %s", s.Config.Implementation, s.GetURL()) + } else { + log.Printf("Using %s S3 implementation for server %s", s.Config.Implementation, s.GetURL()) + } + + // MinIO does not support the setup of bucket CORS rules and MPU abortion lifecycle + if s.Config.Implementation == ImplementationMinio { + s.Config.Setup.CORS = false + s.Config.Setup.AbortIncompleteUploads = 0 + } // Create bucket if it does not exist yet if _, err := s.GetBucketPolicy(&s3.GetBucketPolicyInput{ Bucket: aws.String(s.Config.Bucket), }); err != nil { - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.CreateBucket { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.Setup.CreateBucket { if _, err := s.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(s.Config.Bucket), }); err != nil { @@ -28,7 +38,7 @@ func (s *Server) Setup() error { } // Set CORS configuration for bucket - if s.Implementation != ImplementationMinio { + if s.Config.Setup.CORS { corsRule := &s3.CORSRule{ AllowedHeaders: aws.StringSlice([]string{"Authorization"}), AllowedOrigins: aws.StringSlice([]string{"*"}), @@ -49,56 +59,50 @@ func (s *Server) Setup() error { } } - // Create lifecycle policies - lcRules := []*s3.LifecycleRule{} + if s.Config.Setup.Lifecycle { + // Create lifecycle policies + lcRules := []*s3.LifecycleRule{} - if s.Implementation != ImplementationMinio { - lcRules = append(lcRules, &s3.LifecycleRule{ - ID: aws.String("Abort Multipart Uploads"), - Status: aws.String("Enabled"), - AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{ - DaysAfterInitiation: aws.Int64(31), - }, - Filter: &s3.LifecycleRuleFilter{ - Prefix: aws.String("/"), - }, - }) - } + if s.Config.Setup.AbortIncompleteUploads > 0 { + lcRules = append(lcRules, &s3.LifecycleRule{ + ID: aws.String("Abort Multipart Uploads"), + Status: aws.String("Enabled"), + AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{ + DaysAfterInitiation: aws.Int64(31), + }, + Filter: &s3.LifecycleRuleFilter{ + Prefix: aws.String("/"), + }, + }) + } - for _, cls := range s.Config.Expiration { - lcRules = append(lcRules, &s3.LifecycleRule{ - ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)), - Status: aws.String("Enabled"), - Filter: &s3.LifecycleRuleFilter{ - Tag: &s3.Tag{ - Key: aws.String("expiration"), - Value: aws.String(cls.ID), + for _, cls := range s.Config.Expiration { + lcRules = append(lcRules, &s3.LifecycleRule{ + ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)), + Status: aws.String("Enabled"), + Filter: &s3.LifecycleRuleFilter{ + Tag: &s3.Tag{ + Key: aws.String("expiration"), + Value: aws.String(cls.ID), + }, }, - }, - Expiration: &s3.LifecycleExpiration{ - Days: aws.Int64(cls.Days), - }, - }) - } + Expiration: &s3.LifecycleExpiration{ + Days: aws.Int64(cls.Days), + }, + }) + } - if len(lcRules) > 0 { - if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{ - Bucket: aws.String(s.Config.Bucket), - LifecycleConfiguration: &s3.BucketLifecycleConfiguration{ - Rules: lcRules, - }, - }); err != nil { - return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err) + if len(lcRules) > 0 { + if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{ + Bucket: aws.String(s.Config.Bucket), + LifecycleConfiguration: &s3.BucketLifecycleConfiguration{ + Rules: lcRules, + }, + }); err != nil { + return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err) + } } } - // lc, err := svc.GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{ - // Bucket: aws.String(.Bucket), - // }) - // if err != nil { - // return fmt.Errorf("failed get life-cycle rules: %w", err) - // } - // log.Printf("Life-cycle rules: %+#v\n", lc) - return nil }