[Testing] Reduce flakiness caused by iam bindings (kubeflow#3008)

* Add retry to iam policy bindings

* Add retry for iam policy changes to reduce flakiness

test/deploy-pipeline-lite.sh

@@ -69,13 +69,22 @@ if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then
   export SYSTEM_GSA="test-kfp-system"
   export USER_GSA="test-kfp-user"
 
+  # Workaround for flakiness from gcp-workload-identity-setup.sh:
+  # When two tests add iam policy bindings at the same time, one will fail because
+  # there could be two concurrent changes.
+  # Wait here randomly to reduce chance both scripts are run at the same time
+  # between tests. gcp-workload-identity-setup.sh is user facing, we'd better
+  # not add retry there. Also unless for testing scenario like this, it won't
+  # meet the concurrent change issue.
+  sleep $((RANDOM%30))
   yes | PROJECT_ID=$PROJECT CLUSTER_NAME=$TEST_CLUSTER NAMESPACE=$NAMESPACE \
     ${DIR}/../manifests/kustomize/gcp-workload-identity-setup.sh
 
-  gcloud projects add-iam-policy-binding $PROJECT \
+  source "${DIR}/scripts/retry.sh"
+  retry gcloud projects add-iam-policy-binding $PROJECT \
     --member="serviceAccount:$SYSTEM_GSA@$PROJECT.iam.gserviceaccount.com" \
     --role="roles/editor"
-  gcloud projects add-iam-policy-binding $PROJECT \
+  retry gcloud projects add-iam-policy-binding $PROJECT \
     --member="serviceAccount:$USER_GSA@$PROJECT.iam.gserviceaccount.com" \
     --role="roles/editor"
 

test/install-argo.sh

@@ -58,7 +58,8 @@ if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then
     --member="serviceAccount:$ARGO_GSA@$PROJECT.iam.gserviceaccount.com" \
     --role="roles/editor" \
     > /dev/null # hide verbose output
-  bind_gsa_and_ksa $ARGO_GSA $ARGO_KSA $PROJECT $NAMESPACE
+  source "$DIR/scripts/retry.sh"
+  retry bind_gsa_and_ksa $ARGO_GSA $ARGO_KSA $PROJECT $NAMESPACE
 
   verify_workload_identity_binding $ARGO_KSA $NAMESPACE
 fi

test/scripts/retry.sh

@@ -0,0 +1,23 @@
+# Reference: https://unix.stackexchange.com/a/137639
+
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=1
+  local max=3
+  local delay=2
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}