From c1c239b6be4120d1afdaf83dcf6e4876b6100f20 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Tue, 30 Aug 2022 13:11:53 +0530 Subject: [PATCH] ci: added timeout instead of sleep commands Instead of random sleep time, added timeout so we will wait till the pods get started Closes: https://github.com/rook/kubectl-rook-ceph/issues/42 Signed-off-by: parth-gr --- .github/workflows/ci-for-default-ns.yaml | 20 ++++++++++--- .github/workflows/ci-for-diff-ns.yaml | 36 ++++++++++++++++-------- tests/github-action-helper.sh | 22 +++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-for-default-ns.yaml b/.github/workflows/ci-for-default-ns.yaml index c0af2ef1..355860a9 100644 --- a/.github/workflows/ci-for-default-ns.yaml +++ b/.github/workflows/ci-for-default-ns.yaml @@ -75,15 +75,27 @@ jobs: # for testing start-debug and stop-debug deployment # mons kubectl rook_ceph debug start rook-ceph-mon-a - sleep 5 + tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to exist" 10 \ + "kubectl get pods -nrook-ceph | grep rook-ceph-mon-a-debug-.*Pending'" + debug_mon=$(kubectl get pods -nrook-ceph | grep rook-ceph-mon-a-debug | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to be ready" 10 \ + "kubectl -n rook-ceph get pod $debug_mon | grep 'rook-ceph-mon-a-debug-.*Running'" kubectl rook_ceph debug stop rook-ceph-mon-a # osd kubectl rook_ceph debug start rook-ceph-osd-0 - sleep 5 + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to exist" 10 \ + "kubectl get pods -nrook-ceph | grep rook-ceph-osd-0-debug-.*Pending'" + debug_osd=$(kubectl get pod -n rook-ceph | grep rook-ceph-osd-0-debug | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to be ready" 10 \ + "kubectl -n rook-ceph get pod $debug_osd | grep 'rook-ceph-osd-0-debug-.*Running'" kubectl rook_ceph debug stop rook-ceph-osd-0 - # sleep again so OSD can start - sleep 5 + # check if osd 0 is started + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to exist" 10 \ + "kubectl get pods -nrook-ceph | grep rook-ceph-osd-0.*Pending'" + osd=$(kubectl get pod -n rook-ceph | grep rook-ceph-osd-0 | awk '!/rook-ceph-osd-0-debug/' | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to be ready" 10 \ + "kubectl -n rook-ceph get pod $osd | grep 'rook-ceph-osd-0.*Running'" # for testing osd purge scale the osd deplyment kubectl --namespace rook-ceph scale deploy/rook-ceph-osd-0 --replicas=0 diff --git a/.github/workflows/ci-for-diff-ns.yaml b/.github/workflows/ci-for-diff-ns.yaml index c4e42570..aaf359ff 100644 --- a/.github/workflows/ci-for-diff-ns.yaml +++ b/.github/workflows/ci-for-diff-ns.yaml @@ -49,24 +49,38 @@ jobs: sleep 5 kubectl rook_ceph -o test-operator -n test-cluster rbd ls replicapool - # for testing osd purge scale the osd deplyment - kubectl --namespace test-cluster scale deploy/rook-ceph-osd-0 --replicas=0 - # we need to sleep so the osd will be marked down before purging the osd - sleep 5 - kubectl-rook_ceph -o test-operator -n test-cluster rook purge-osd 0 --force - kubectl rook_ceph -o test-operator -n test-cluster health - # for testing start-debug and stop-debug deployment # mon kubectl rook_ceph -o test-operator -n test-cluster debug start rook-ceph-mon-a - # sleep till the debug pod get running - sleep 5 + tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to exist" 10 \ + "kubectl get pods -n test-cluster -o test-operator | grep rook-ceph-mon-a-debug-.*Pending'" + debug_mon=$(kubectl get pods -ntest-cluster | grep rook-ceph-mon-a-debug | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-mon-a-debug deployment pod to be ready" 10 \ + "kubectl -n test-cluster get pod $debug_mon | grep 'rook-ceph-mon-a-debug-.*Running'" kubectl rook_ceph -o test-operator -n test-cluster debug stop rook-ceph-mon-a # osd kubectl rook_ceph -o test-operator -n test-cluster debug start rook-ceph-osd-0 - # sleep till the debug pod get running - sleep 5 + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to exist" 10 \ + "kubectl get pods -n test-cluster -o test-operator | grep rook-ceph-osd-0-debug-.*Pending'" + debug_osd=$(kubectl get pod -n test-cluster | grep rook-ceph-osd-0-debug | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0-debug deployment pod to be ready" 10 \ + "kubectl -n test-cluster get pod $debug_osd | grep 'rook-ceph-osd-0-debug-.*Running'" kubectl rook_ceph -o test-operator -n test-cluster debug stop rook-ceph-osd-0 + + # check if osd 0 is started + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to exist" 10 \ + "kubectl get pods -n test-cluster -o test-operator | grep rook-ceph-osd-0.*Pending'" + osd=$(kubectl get pod -n test-cluster | grep rook-ceph-osd-0 | awk '!/rook-ceph-osd-0-debug/' | awk '{ print $1 }') + tests/github-action-helper.sh wait_for_condition "rook-ceph-osd-0 deployment pod to be ready" 10 \ + "kubectl -n test-cluster get pod $osd | grep 'rook-ceph-osd-0.*Running'" + + # for testing osd purge scale the osd deplyment + kubectl --namespace test-cluster scale deploy/rook-ceph-osd-0 --replicas=0 + # we need to sleep so the osd will be marked down before purging the osd + sleep 5 + kubectl-rook_ceph -o test-operator -n test-cluster rook purge-osd 0 --force + kubectl rook_ceph -o test-operator -n test-cluster health + - name: setup tmate session for debugging when event is PR if: failure() && github.event_name == 'pull_request' diff --git a/tests/github-action-helper.sh b/tests/github-action-helper.sh index f521c7f9..cd0297d2 100755 --- a/tests/github-action-helper.sh +++ b/tests/github-action-helper.sh @@ -104,6 +104,28 @@ wait_for_operator_pod_to_be_ready_state_custom() { EOF } +wait_for_condition() { + REASON="$1" # reason + TIMEOUT="$2" # in seconds + CMD="$3" + + BASH_CMD="${BASH_CMD:=bash}" # might be 'bash -x', for example + + msg="Waiting ${TIMEOUT} seconds for ${REASON} ..." + echo "${msg} countdown: ${TIMEOUT}" + start_time=$SECONDS + until ${BASH_CMD} -c "${CMD}" 2>&1 ; do + echo "" + if (( SECONDS - start_time >= TIMEOUT )); then + echo "${msg} timed out! (output above)" + exit 1 + fi + echo "${msg} countdown: $((TIMEOUT - SECONDS + start_time)) " + sleep 5 + done + echo "${msg} done after $((SECONDS - start_time)) seconds." +} + ######## # MAIN # ########