From 5c4cda5221499d2e66e94f6105c8f559dc702777 Mon Sep 17 00:00:00 2001
From: Sameer Naik <sameer@bitnami.com>
Date: Fri, 22 Mar 2019 10:26:02 +0530
Subject: [PATCH] ci: retry failed stages only once after a failure

Retrying failed build stages 3 times is excessive and wasteful in most
cases. To account for transient cluster creation failures, one
additional retry should be sufficient to overcome issues coming from
cluster creation issues.

Additionally we have added a 3 minute wait before retrying.
---
 Jenkinsfile | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index aeefd8283a..55d3a6ee12 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -277,7 +277,7 @@ spec:
                     }, failFast: true
                 )
 
-                def maxRetries = 3
+                def maxRetries = 2
                 def platforms = [:]
 
                 // See:
@@ -291,11 +291,16 @@ spec:
                     platforms[platform] = {
                         stage(platform) {
                             def retryNum = 0
+
                             retry(maxRetries) {
                                 def clusterName = ("${env.BRANCH_NAME}".take(8) + "-${env.BUILD_NUMBER}-" + UUID.randomUUID().toString().take(5) + "-${platform}").replaceAll(/[^a-zA-Z0-9-]/, '-').replaceAll(/--/, '-').toLowerCase()
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv(["KUBECONFIG=${env.WORKSPACE}/.kubecfg-${clusterName}"]) {
@@ -385,6 +390,10 @@ spec:
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv(["KUBECONFIG=${env.WORKSPACE}/.kubecfg-${clusterName}"]) {
@@ -502,6 +511,10 @@ spec:
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv([