ci: added timeout instead of sleep commands

Instead of random sleep time, added timeout so we will wait till the pods get started Closes: rook#42 Signed-off-by: parth-gr <paarora@redhat.com>
parth-gr · Sep 20, 2022 · c1c239b · c1c239b
1 parent a76fc34
commit c1c239b
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 15 deletions.
diff --git a/.github/workflows/ci-for-default-ns.yaml b/.github/workflows/ci-for-default-ns.yaml
@@ -75,15 +75,27 @@ jobs:
           # for testing start-debug and stop-debug deployment
           # mons
           kubectl rook_ceph debug start rook-ceph-mon-a
-          sleep 5
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to exist" 10 \
+            "kubectl get pods -nrook-ceph | grep rook-ceph-mon-a-debug-.*Pending'"
+          debug_mon=$(kubectl get pods -nrook-ceph | grep rook-ceph-mon-a-debug | awk '{ print $1  }')          
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to be ready" 10 \
+            "kubectl -n rook-ceph get pod $debug_mon | grep 'rook-ceph-mon-a-debug-.*Running'"
           kubectl rook_ceph debug stop rook-ceph-mon-a
           # osd
           kubectl rook_ceph debug start rook-ceph-osd-0
-          sleep 5
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to exist" 10 \
+            "kubectl get pods -nrook-ceph | grep rook-ceph-osd-0-debug-.*Pending'"
+          debug_osd=$(kubectl get pod -n rook-ceph | grep rook-ceph-osd-0-debug | awk '{ print $1  }')
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to be ready" 10 \
+            "kubectl -n rook-ceph get pod $debug_osd | grep 'rook-ceph-osd-0-debug-.*Running'"
           kubectl rook_ceph debug stop rook-ceph-osd-0
 
-          # sleep again so OSD can start
-          sleep 5
+          # check if osd 0 is started
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to exist" 10 \
+            "kubectl get pods -nrook-ceph | grep rook-ceph-osd-0.*Pending'"
+          osd=$(kubectl get pod -n rook-ceph | grep rook-ceph-osd-0 | awk '!/rook-ceph-osd-0-debug/' | awk '{ print $1  }')
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to be ready" 10 \
+            "kubectl -n rook-ceph get pod $osd | grep 'rook-ceph-osd-0.*Running'"
 
           # for testing osd purge scale the osd deplyment
           kubectl --namespace rook-ceph scale deploy/rook-ceph-osd-0 --replicas=0

diff --git a/.github/workflows/ci-for-diff-ns.yaml b/.github/workflows/ci-for-diff-ns.yaml
@@ -49,24 +49,38 @@ jobs:
           sleep 5
           kubectl rook_ceph -o test-operator -n test-cluster rbd ls replicapool
 
-          # for testing osd purge scale the osd deplyment
-          kubectl --namespace test-cluster scale deploy/rook-ceph-osd-0 --replicas=0
-          # we need to sleep so the osd will be marked down before purging the osd
-          sleep 5
-          kubectl-rook_ceph -o test-operator -n test-cluster rook purge-osd 0 --force
-          kubectl rook_ceph -o test-operator -n test-cluster health
-
           # for testing start-debug and stop-debug deployment
           # mon
           kubectl rook_ceph -o test-operator -n test-cluster debug start rook-ceph-mon-a
-          # sleep till the debug pod get running
-          sleep 5
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to exist" 10 \
+            "kubectl get pods  -n test-cluster -o test-operator | grep rook-ceph-mon-a-debug-.*Pending'"
+          debug_mon=$(kubectl get pods -ntest-cluster | grep rook-ceph-mon-a-debug | awk '{ print $1  }')  
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to be ready" 10 \
+            "kubectl -n test-cluster get pod $debug_mon | grep 'rook-ceph-mon-a-debug-.*Running'"
           kubectl rook_ceph -o test-operator -n test-cluster debug stop rook-ceph-mon-a
           # osd
           kubectl rook_ceph -o test-operator -n test-cluster debug start rook-ceph-osd-0
-          # sleep till the debug pod get running
-          sleep 5
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to exist" 10 \
+            "kubectl get pods  -n test-cluster -o test-operator | grep rook-ceph-osd-0-debug-.*Pending'"
+          debug_osd=$(kubectl get pod -n test-cluster | grep rook-ceph-osd-0-debug | awk '{ print $1  }')
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to be ready" 10 \
+            "kubectl -n test-cluster get pod $debug_osd | grep 'rook-ceph-osd-0-debug-.*Running'"
           kubectl rook_ceph -o test-operator -n test-cluster debug stop rook-ceph-osd-0
+          
+          # check if osd 0 is started
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to exist" 10 \
+            "kubectl get pods  -n test-cluster -o test-operator | grep rook-ceph-osd-0.*Pending'"
+          osd=$(kubectl get pod -n test-cluster | grep rook-ceph-osd-0 | awk '!/rook-ceph-osd-0-debug/' | awk '{ print $1  }')
+          tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to be ready" 10 \
+            "kubectl -n test-cluster get pod $osd | grep 'rook-ceph-osd-0.*Running'"
+
+          # for testing osd purge scale the osd deplyment
+          kubectl --namespace test-cluster scale deploy/rook-ceph-osd-0 --replicas=0
+          # we need to sleep so the osd will be marked down before purging the osd
+          sleep 5
+          kubectl-rook_ceph -o test-operator -n test-cluster rook purge-osd 0 --force
+          kubectl rook_ceph -o test-operator -n test-cluster health
+
 
       - name: setup tmate session for debugging when event is PR
         if: failure() && github.event_name == 'pull_request'

diff --git a/tests/github-action-helper.sh b/tests/github-action-helper.sh
@@ -104,6 +104,28 @@ wait_for_operator_pod_to_be_ready_state_custom() {
 EOF
 }
 
+wait_for_condition() {
+  REASON="$1"  # reason
+  TIMEOUT="$2"  # in seconds
+  CMD="$3"
+
+  BASH_CMD="${BASH_CMD:=bash}" # might be 'bash -x', for example
+
+  msg="Waiting ${TIMEOUT} seconds for ${REASON} ..."
+  echo "${msg} countdown: ${TIMEOUT}"
+  start_time=$SECONDS
+  until ${BASH_CMD} -c "${CMD}" 2>&1 ; do
+    echo ""
+    if (( SECONDS - start_time >= TIMEOUT )); then
+      echo "${msg} timed out! (output above)"
+      exit 1
+    fi
+    echo "${msg} countdown: $((TIMEOUT - SECONDS + start_time)) "
+    sleep 5
+  done
+  echo "${msg} done after $((SECONDS - start_time)) seconds."
+}
+
 ########
 # MAIN #
 ########