make detection of S3 implementation more robust and handle slowly sta…

…rting S3 servers like MinIO

pkg/server/retryer.go

@@ -0,0 +1,26 @@
+package server
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+type retryer struct {
+	client.DefaultRetryer
+}
+
+func (d retryer) RetryRules(r *request.Request) time.Duration {
+	ra := r.HTTPResponse.Header.Get("Retry-after")
+	svr := r.HTTPResponse.Header.Get("Server")
+
+	// MinIO returns a "Retry-after: 120" header when the server is still initializing
+	// Waiting 120 seconds is far too long as most servers finish their initialization faster
+	// We reduce the time to 1 second and let the other retry rules handle the rest.
+	if ra != "" && svr == "MinIO" && r.HTTPResponse.StatusCode == 503 {
+		r.HTTPResponse.Header.Set("Retry-after", "1")
+	}
+
+	return d.DefaultRetryer.RetryRules(r)
+}

pkg/server/server.go

@@ -1,11 +1,11 @@
 package server
 
 import (
-	"net/http"
 	"net/url"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/stv0g/gose/pkg/config"
 )
@@ -75,17 +75,21 @@ func (s *Server) DetectImplementation() Implementation {
 	} else if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") {
 		return ImplementationGoogleCloudStorage
 	} else {
-		u := s.GetObjectURL("not-existing")
-
-		if r, err := http.Get(u.String()); err != nil {
-			return ImplementationUnknown
-		} else {
-			if svr := r.Header.Get("Server"); svr == "" {
-				return ImplementationUnknown
-			} else {
+		req, _ := s.S3.ListBucketsRequest(&s3.ListBucketsInput{})
+		req.Retryer = retryer{
+			DefaultRetryer: client.DefaultRetryer{
+				NumMaxRetries: 10,
+			},
+		}
+		if err := req.Send(); err == nil {
+			if svr := req.HTTPResponse.Header.Get("Server"); svr != "" {
 				return Implementation(svr)
 			}
+
+			return ImplementationUnknown
 		}
+
+		return ImplementationUnknown
 	}
 }
 

pkg/server/setup.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"log"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -11,6 +12,7 @@ import (
 // Setup initializes the S3 bucket (life-cycle rules & CORS)
 func (s *Server) Setup() error {
 	s.Implementation = s.DetectImplementation()
+	log.Printf("Detected %s S3 implementation for server %s\n", s.Implementation, s.GetURL())
 
 	// Create bucket if it does not exist yet
 	if _, err := s.GetBucketPolicy(&s3.GetBucketPolicyInput{