ci: retry failed stages only once after a failure

Retrying failed build stages 3 times is excessive and wasteful in most
cases. To account for transient cluster creation failures, one
additional retry should be sufficient to overcome issues coming from
cluster creation issues.

Additionally we have added a 3 minute wait before retrying.

Jenkinsfile

@@ -277,7 +277,7 @@ spec:
                     }, failFast: true
                 )
 
-                def maxRetries = 3
+                def maxRetries = 2
                 def platforms = [:]
 
                 // See:
@@ -291,11 +291,16 @@ spec:
                     platforms[platform] = {
                         stage(platform) {
                             def retryNum = 0
+
                             retry(maxRetries) {
                                 def clusterName = ("${env.BRANCH_NAME}".take(8) + "-${env.BUILD_NUMBER}-" + UUID.randomUUID().toString().take(5) + "-${platform}").replaceAll(/[^a-zA-Z0-9-]/, '-').replaceAll(/--/, '-').toLowerCase()
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv(["KUBECONFIG=${env.WORKSPACE}/.kubecfg-${clusterName}"]) {
@@ -385,6 +390,10 @@ spec:
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv(["KUBECONFIG=${env.WORKSPACE}/.kubecfg-${clusterName}"]) {
@@ -502,6 +511,10 @@ spec:
                                 def adminEmail = "${clusterName}@${parentZone}"
                                 def dnsZone = "${clusterName}.${parentZone}"
 
+                                if (retryNum > 0) {
+                                    sleep 180
+                                }
+
                                 retryNum++
                                 dir("${env.WORKSPACE}/${clusterName}") {
                                     withEnv([