Merge pull request #2667 from nkubala/gcb-quota

Implement exponential backoff for retrieving cloud build status

pkg/skaffold/build/gcb/cloud_build.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	cstorage "cloud.google.com/go/storage"
@@ -40,6 +41,7 @@ import (
 	cloudbuild "google.golang.org/api/cloudbuild/v1"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Build builds a list of artifacts with Google Cloud Build.
@@ -112,9 +114,25 @@ func (b *Builder) buildArtifactWithCloudBuild(ctx context.Context, out io.Writer
 	offset := int64(0)
 watch:
 	for {
+		var cb *cloudbuild.Build
+		var err error
 		logrus.Debugf("current offset %d", offset)
-		cb, err := cbclient.Projects.Builds.Get(projectID, remoteID).Do()
-		if err != nil {
+		backoff := NewStatusBackoff()
+		if waitErr := wait.Poll(backoff.Duration, RetryTimeout, func() (bool, error) {
+			backoff.Step()
+			cb, err = cbclient.Projects.Builds.Get(projectID, remoteID).Do()
+			if err == nil {
+				return true, nil
+			}
+			if strings.Contains(err.Error(), "Error 429: Quota exceeded for quota metric 'cloudbuild.googleapis.com/get_requests'") {
+				// if we hit the rate limit, continue to retry
+				return false, nil
+			}
+			return false, err
+		}); waitErr != nil {
+			return "", errors.Wrap(waitErr, "getting build status")
+		}
+		if cb == nil {
 			return "", errors.Wrap(err, "getting build status")
 		}
 

pkg/skaffold/build/gcb/types.go

@@ -29,6 +29,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -58,8 +59,26 @@ const (
 
 	// RetryDelay is the time to wait in between polling the status of the cloud build
 	RetryDelay = 1 * time.Second
+
+	// BackoffFactor is the exponent for exponential backoff during build status polling
+	BackoffFactor = 1.5
+
+	// BackoffSteps is the number of times we increase the backoff time during exponential backoff
+	BackoffSteps = 10
+
+	// RetryTimeout is the max amount of time to retry getting the status of the build before erroring
+	RetryTimeout = 3 * time.Minute
 )
 
+func NewStatusBackoff() *wait.Backoff {
+	return &wait.Backoff{
+		Duration: RetryDelay,
+		Factor:   float64(BackoffFactor),
+		Steps:    BackoffSteps,
+		Cap:      60 * time.Second,
+	}
+}
+
 // Builder builds artifacts with Google Cloud Build.
 type Builder struct {
 	*latest.GoogleCloudBuild