fix(csi): prevent race condition in ci tests step

csi/test/e2e_test.sh

@@ -6,16 +6,10 @@ set -o xtrace
 # This test assumes there is a Kubernetes environment up and running.
 # It could be either a remote one or a local one (e.g., using KinD or minikube).
 
-# Function to check if the port is ready
-wait_for_port() {
-  local port=$1
-  while ! nc -z localhost $port; do
-    sleep 0.1
-  done
-}
-
 DIR="$(dirname "$0")"
 
+source ./${DIR}/test_utils.sh
+
 KUBECTL=${KUBECTL:-"kubectl"}
 
 # You can provide a local version of the model registry storage initializer
@@ -145,7 +139,7 @@ spec:
 EOF
 
 # wait for pod predictor to be initialized
-sleep 2
+repeat_cmd_until "kubectl get pod -n $KSERVE_TEST_NAMESPACE --selector='component=predictor' | wc -l" "-gt 0" 60
 predictor=$(kubectl get pod -n $KSERVE_TEST_NAMESPACE --selector="component=predictor" --output jsonpath='{.items[0].metadata.name}')
 kubectl wait --for=condition=Ready pod/$predictor -n $KSERVE_TEST_NAMESPACE --timeout=5m
 

csi/test/test_utils.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# Function to check if the port is ready
+wait_for_port() {
+  local port=$1
+  while ! nc -z localhost $port; do
+    sleep 0.1
+  done
+}
+
+repeat_cmd_until() {
+  local cmd=$1
+  local condition=$2
+  local max_wait_secs=$3
+  local interval_secs=2
+  local start_time=$(date +%s)
+  local output
+
+  while true; do
+
+    current_time=$(date +%s)
+    if (( (current_time - start_time) > max_wait_secs )); then
+      echo "Waited for expression "$1" to satisfy condition "$2" for $max_wait_secs seconds without luck. Returning with error."
+      return 1
+    fi
+
+    output=$(eval $cmd)
+
+    if [ $output $condition ]; then
+      break
+    else
+      sleep $interval_secs
+    fi
+  done
+}