ci: download apache-zookeeper-3.7.0-bin.tar.gz during release buildin…

…g for unit tests once third-parties have change (#1186)
apache · Oct 20, 2022 · 359500e · 359500e
1 parent d3e9cb8
commit 359500e
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 34 deletions.
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
@@ -94,6 +94,7 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
           cmake --build build/ -j $(nproc)
           ../scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_zk.sh zookeeper-bin
       - name: Compilation
         run: |
           ccache -p
@@ -107,8 +108,9 @@ jobs:
       - name: Tar files
         run: |
           mv thirdparty/hadoop-bin ./
+          mv thirdparty/zookeeper-bin ./
           rm -rf thirdparty
-          tar -zcvhf release__builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+          tar -zcvhf release__builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
       - name: Upload Artifact
         uses: actions/upload-artifact@v3
         with:
@@ -212,6 +214,7 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
           cmake --build build/ -j $(nproc)
           ../scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_zk.sh zookeeper-bin
       - name: Compilation
         run: |
           ccache -p
@@ -221,8 +224,9 @@ jobs:
       - name: Tar files
         run: |
           mv thirdparty/hadoop-bin ./
+          mv thirdparty/zookeeper-bin ./
           rm -rf thirdparty
-          tar -zcvhf release_address_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+          tar -zcvhf release_address_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
       - name: Upload Artifact
         uses: actions/upload-artifact@v3
         with:
@@ -331,6 +335,7 @@ jobs:
 #          cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
 #          cmake --build build/ -j $(nproc)
 #          ../scripts/download_hadoop.sh hadoop-bin
+#          ../scripts/download_zk.sh zookeeper-bin
 #      - name: Compilation
 #        run: |
 #          ccache -p
@@ -340,8 +345,9 @@ jobs:
 #      - name: Tar files
 #        run: |
 #          mv thirdparty/hadoop-bin ./
+#          mv thirdparty/zookeeper-bin ./
 #          rm -rf thirdparty
-#          tar -zcvhf release_undefined_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+#          tar -zcvhf release_undefined_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
 #      - name: Upload Artifact
 #        uses: actions/upload-artifact@v3
 #        with:
@@ -443,6 +449,7 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
           cmake --build build/ -j $(nproc)
           ../scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_zk.sh zookeeper-bin
       - name: Compilation
         run: |
           ccache -p
@@ -456,8 +463,9 @@ jobs:
       - name: Tar files
         run: |
           mv thirdparty/hadoop-bin ./
+          mv thirdparty/zookeeper-bin ./
           rm -rf thirdparty
-          tar -zcvhf release_jemalloc_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+          tar -zcvhf release_jemalloc_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
       - name: Upload Artifact
         uses: actions/upload-artifact@v3
         with:

diff --git a/run.sh b/run.sh
@@ -491,6 +491,13 @@ function run_start_zk()
     type nc >/dev/null 2>&1 || { echo >&2 "start zk failed, need install netcat command..."; exit 1;}
 
     INSTALL_DIR=`pwd`/.zk_install
+    if [ ! -d "${INSTALL_DIR}/zookeeper-bin" ]; then
+        if [ -d "zookeeper-bin" ]; then
+            # this zookeeper-bin must have been got from github action workflows, thus just
+            # move it to ${INSTALL_DIR} to prevent from downloading
+            mv zookeeper-bin ${INSTALL_DIR}/
+        fi
+    fi
     PORT=22181
     while [[ $# > 0 ]]; do
         key="$1"

diff --git a/scripts/clear_zk.sh b/scripts/clear_zk.sh
@@ -34,11 +34,20 @@ then
 fi
 
 cd $INSTALL_DIR
+
+# If the old dir for zk bin exists, just use it
 ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
 
-if [ -d "$ZOOKEEPER_HOME" ]
-then
-    $ZOOKEEPER_HOME/bin/zkServer.sh stop
-    rm -rf $ZOOKEEPER_HOME/data &>/dev/null
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+    ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
+    rm -rf ${ZOOKEEPER_HOME} &> /dev/null
+    echo "Deleting old zookeeper ... DELETED"
+fi
+
+ZOOKEEPER_HOME=`pwd`/zookeeper-bin
+
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+    ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
+    rm -rf ${ZOOKEEPER_HOME}/data &> /dev/null
     echo "Clearing zookeeper ... CLEARED"
 fi
diff --git a/scripts/download_package.sh b/scripts/download_package.sh
@@ -71,4 +71,9 @@ if [ ! -d ${DIR_NAME} ]; then
     exit 1
 fi
 
+if [ -d ${TARGET_PATH} ]; then
+    echo "TARGET_PATH ${TARGET_PATH} has been generated, which means it and ${DIR_NAME} are the same dir thus do not do mv any more"
+    exit 0
+fi
+
 mv ${DIR_NAME} ${TARGET_PATH}
diff --git a/scripts/start_zk.sh b/scripts/start_zk.sh
@@ -18,6 +18,8 @@
 
 set -e
 
+CWD=$(cd $(dirname $0) && pwd)
+
 # Options:
 #    INSTALL_DIR    <dir>
 #    PORT           <port>
@@ -42,32 +44,14 @@ fi
 
 cd "$INSTALL_DIR" || exit
 
-ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin
-ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz
-ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a"
+# If the old dir for zk bin exists, just use it
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
 
-if [ ! -f $ZOOKEEPER_TAR_NAME ]; then
-    echo "Downloading zookeeper..."
-    download_url="http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz"
-    if ! wget -T 10 -t 5 $download_url; then
-        echo "ERROR: download zookeeper failed"
-        exit 1
-    fi
-    if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` != $ZOOKEEPER_TAR_MD5_VALUE ]; then
-        echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!"
-        exit 1
-    fi
-fi
-
-if [ ! -d $ZOOKEEPER_ROOT ]; then
-    echo "Decompressing zookeeper..."
-    if ! tar xf $ZOOKEEPER_TAR_NAME; then
-        echo "ERROR: decompress zookeeper failed"
-        exit 1
-    fi
+if [ ! -d "${ZOOKEEPER_HOME}" ]; then
+    ${CWD}/download_zk.sh zookeeper-bin
+    ZOOKEEPER_HOME=`pwd`/zookeeper-bin
 fi
 
-ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT
 ZOOKEEPER_PORT=$PORT
 
 cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg

diff --git a/scripts/stop_zk.sh b/scripts/stop_zk.sh
@@ -33,9 +33,14 @@ then
 fi
 
 cd $INSTALL_DIR
+
+# If the old dir for zk bin exists, just use it
 ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
 
-if [ -d "$ZOOKEEPER_HOME" ]
-then
-    $ZOOKEEPER_HOME/bin/zkServer.sh stop
+if [ ! -d "${ZOOKEEPER_HOME}" ]; then
+    ZOOKEEPER_HOME=`pwd`/zookeeper-bin
+fi
+
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+    ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
 fi