From 093709ca00b7ae8df00ceb4a112e6826fa2e8a77 Mon Sep 17 00:00:00 2001
From: Nick Kubala <kubala.nick@gmail.com>
Date: Thu, 15 Aug 2019 16:17:42 -0700
Subject: [PATCH 1/2] implement exponential backoff for retrieving cloud build
 status

---
 pkg/skaffold/build/gcb/cloud_build.go | 22 ++++++++++++++++++++--
 pkg/skaffold/build/gcb/types.go       | 19 +++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/pkg/skaffold/build/gcb/cloud_build.go b/pkg/skaffold/build/gcb/cloud_build.go
index 1643a06ee43..cb9e17e9a72 100644
--- a/pkg/skaffold/build/gcb/cloud_build.go
+++ b/pkg/skaffold/build/gcb/cloud_build.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	cstorage "cloud.google.com/go/storage"
@@ -40,6 +41,7 @@ import (
 	cloudbuild "google.golang.org/api/cloudbuild/v1"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Build builds a list of artifacts with Google Cloud Build.
@@ -112,9 +114,25 @@ func (b *Builder) buildArtifactWithCloudBuild(ctx context.Context, out io.Writer
 	offset := int64(0)
 watch:
 	for {
+		var cb *cloudbuild.Build
+		var err error
 		logrus.Debugf("current offset %d", offset)
-		cb, err := cbclient.Projects.Builds.Get(projectID, remoteID).Do()
-		if err != nil {
+		backoff := NewStatusBackoff()
+		if waitErr := wait.Poll(backoff.Duration, RetryTimeout, func() (bool, error) {
+			backoff.Step()
+			cb, err = cbclient.Projects.Builds.Get(projectID, remoteID).Do()
+			if err == nil {
+				return true, nil
+			}
+			if strings.Contains(err.Error(), "Quota exceeded") {
+				// if we hit the rate limit, continue to retry
+				return false, nil
+			}
+			return false, err
+		}); waitErr != nil {
+			return "", errors.Wrap(waitErr, "getting build status")
+		}
+		if cb == nil {
 			return "", errors.Wrap(err, "getting build status")
 		}
 
diff --git a/pkg/skaffold/build/gcb/types.go b/pkg/skaffold/build/gcb/types.go
index c6b73f82824..bbca7602148 100644
--- a/pkg/skaffold/build/gcb/types.go
+++ b/pkg/skaffold/build/gcb/types.go
@@ -29,6 +29,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -58,8 +59,26 @@ const (
 
 	// RetryDelay is the time to wait in between polling the status of the cloud build
 	RetryDelay = 1 * time.Second
+
+	// BackoffFactor is the exponent for exponential backoff during build status polling
+	BackoffFactor = 1.5
+
+	// BackoffSteps is the number of times we increase the backoff time during exponential backoff
+	BackoffSteps = 10
+
+	// RetryTimeout is the max amount of time to retry getting the status of the build before erroring
+	RetryTimeout = 3 * time.Minute
 )
 
+func NewStatusBackoff() *wait.Backoff {
+	return &wait.Backoff{
+		Duration: RetryDelay,
+		Factor:   float64(BackoffFactor),
+		Steps:    BackoffSteps,
+		Cap:      60 * time.Second,
+	}
+}
+
 // Builder builds artifacts with Google Cloud Build.
 type Builder struct {
 	*latest.GoogleCloudBuild

From 70c97ce2edf6060103bfcc317d91addedd450c34 Mon Sep 17 00:00:00 2001
From: Nick Kubala <kubala.nick@gmail.com>
Date: Mon, 19 Aug 2019 15:31:45 -0700
Subject: [PATCH 2/2] more specific error message matching

---
 pkg/skaffold/build/gcb/cloud_build.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/skaffold/build/gcb/cloud_build.go b/pkg/skaffold/build/gcb/cloud_build.go
index cb9e17e9a72..0bb7262db4e 100644
--- a/pkg/skaffold/build/gcb/cloud_build.go
+++ b/pkg/skaffold/build/gcb/cloud_build.go
@@ -124,7 +124,7 @@ watch:
 			if err == nil {
 				return true, nil
 			}
-			if strings.Contains(err.Error(), "Quota exceeded") {
+			if strings.Contains(err.Error(), "Error 429: Quota exceeded for quota metric 'cloudbuild.googleapis.com/get_requests'") {
 				// if we hit the rate limit, continue to retry
 				return false, nil
 			}