From 1746217ad19f879f9da2647b132e0e0103945ab2 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 13 Oct 2022 15:08:51 +0800 Subject: [PATCH 1/3] ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have changed --- .github/workflows/lint_and_test_cpp.yaml | 16 +++++++++---- run.sh | 5 ++++ scripts/clear_zk.sh | 15 ++++++++---- scripts/download_package.sh | 5 ++++ scripts/start_zk.sh | 29 +++++------------------- scripts/stop_zk.sh | 9 +++++--- 6 files changed, 45 insertions(+), 34 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index bd90db09b2..cd717c9575 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -94,6 +94,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ cmake --build build/ -j $(nproc) ../scripts/download_hadoop.sh hadoop-bin + ../scripts/download_zk.sh zookeeper-bin - name: Compilation run: | ccache -p @@ -107,8 +108,9 @@ jobs: - name: Tar files run: | mv thirdparty/hadoop-bin ./ + mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*' + tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*' - name: Upload Artifact uses: actions/upload-artifact@v3 with: @@ -210,6 +212,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ cmake --build build/ -j $(nproc) ../scripts/download_hadoop.sh hadoop-bin + ../scripts/download_zk.sh zookeeper-bin - name: Compilation run: | ccache -p @@ -219,8 +222,9 @@ jobs: - name: Tar files run: | mv thirdparty/hadoop-bin ./ + mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - tar -zcvhf release_address_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*' + tar -zcvhf release_address_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*' - name: Upload Artifact uses: actions/upload-artifact@v3 with: @@ -322,6 +326,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/ cmake --build build/ -j $(nproc) ../scripts/download_hadoop.sh hadoop-bin + ../scripts/download_zk.sh zookeeper-bin - name: Compilation run: | ccache -p @@ -331,8 +336,9 @@ jobs: - name: Tar files run: | mv thirdparty/hadoop-bin ./ + mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - tar -zcvhf release_undefined_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*' + tar -zcvhf release_undefined_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*' - name: Upload Artifact uses: actions/upload-artifact@v3 with: @@ -434,6 +440,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ cmake --build build/ -j $(nproc) ../scripts/download_hadoop.sh hadoop-bin + ../scripts/download_zk.sh zookeeper-bin - name: Compilation run: | ccache -p @@ -447,8 +454,9 @@ jobs: - name: Tar files run: | mv thirdparty/hadoop-bin ./ + mv thirdparty/zookeeper-bin ./ rm -rf thirdparty - tar -zcvhf release_jemalloc_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*' + tar -zcvhf release_jemalloc_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*' - name: Upload Artifact uses: actions/upload-artifact@v3 with: diff --git a/run.sh b/run.sh index 67e774cc94..a9ef41ba22 100755 --- a/run.sh +++ b/run.sh @@ -492,6 +492,11 @@ function run_start_zk() type nc >/dev/null 2>&1 || { echo >&2 "start zk failed, need install netcat command..."; exit 1;} INSTALL_DIR=`pwd`/.zk_install + if [ -d "zookeeper-bin" ]; then + # this zookeeper-bin must have been got from github action workflows, thus just + # move it to ${INSTALL_DIR} to prevent from downloading + mv zookeeper-bin ${INSTALL_DIR}/ + fi PORT=22181 while [[ $# > 0 ]]; do key="$1" diff --git a/scripts/clear_zk.sh b/scripts/clear_zk.sh index c94cbe03a0..11267e73c8 100755 --- a/scripts/clear_zk.sh +++ b/scripts/clear_zk.sh @@ -36,9 +36,16 @@ fi cd $INSTALL_DIR ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin -if [ -d "$ZOOKEEPER_HOME" ] -then - $ZOOKEEPER_HOME/bin/zkServer.sh stop - rm -rf $ZOOKEEPER_HOME/data &>/dev/null +if [ -d "${ZOOKEEPER_HOME}" ]; then + ${ZOOKEEPER_HOME}/bin/zkServer.sh stop + rm -rf ${ZOOKEEPER_HOME} &> /dev/null + echo "Deleting old zookeeper ... DELETED" +fi + +ZOOKEEPER_HOME=`pwd`/zookeeper-bin + +if [ -d "${ZOOKEEPER_HOME}" ]; then + ${ZOOKEEPER_HOME}/bin/zkServer.sh stop + rm -rf ${ZOOKEEPER_HOME}/data &> /dev/null echo "Clearing zookeeper ... CLEARED" fi diff --git a/scripts/download_package.sh b/scripts/download_package.sh index bc9202bfa6..5383934023 100755 --- a/scripts/download_package.sh +++ b/scripts/download_package.sh @@ -71,4 +71,9 @@ if [ ! -d ${DIR_NAME} ]; then exit 1 fi +if [ -d ${TARGET_PATH} ]; then + echo "TARGET_PATH ${TARGET_PATH} has been generated, which means it and ${DIR_NAME} are the same dir thus do not do mv any more" + exit 0 +fi + mv ${DIR_NAME} ${TARGET_PATH} diff --git a/scripts/start_zk.sh b/scripts/start_zk.sh index 0811652ffd..d23c1e89a9 100755 --- a/scripts/start_zk.sh +++ b/scripts/start_zk.sh @@ -18,6 +18,8 @@ set -e +CWD=$(cd $(dirname $0) && pwd) + # Options: # INSTALL_DIR # PORT @@ -42,32 +44,13 @@ fi cd "$INSTALL_DIR" || exit -ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin -ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz -ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a" +ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin -if [ ! -f $ZOOKEEPER_TAR_NAME ]; then - echo "Downloading zookeeper..." - download_url="http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz" - if ! wget -T 10 -t 5 $download_url; then - echo "ERROR: download zookeeper failed" - exit 1 - fi - if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` != $ZOOKEEPER_TAR_MD5_VALUE ]; then - echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!" - exit 1 - fi -fi - -if [ ! -d $ZOOKEEPER_ROOT ]; then - echo "Decompressing zookeeper..." - if ! tar xf $ZOOKEEPER_TAR_NAME; then - echo "ERROR: decompress zookeeper failed" - exit 1 - fi +if [ ! -d "${ZOOKEEPER_HOME}" ]; then + ${CWD}/download_zk.sh zookeeper-bin + ZOOKEEPER_HOME=`pwd`/zookeeper-bin fi -ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT ZOOKEEPER_PORT=$PORT cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg diff --git a/scripts/stop_zk.sh b/scripts/stop_zk.sh index 313fd4f547..0b4c45cbe2 100755 --- a/scripts/stop_zk.sh +++ b/scripts/stop_zk.sh @@ -35,7 +35,10 @@ fi cd $INSTALL_DIR ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin -if [ -d "$ZOOKEEPER_HOME" ] -then - $ZOOKEEPER_HOME/bin/zkServer.sh stop +if [ ! -d "${ZOOKEEPER_HOME}" ]; then + ZOOKEEPER_HOME=`pwd`/zookeeper-bin +fi + +if [ -d "${ZOOKEEPER_HOME}" ]; then + ${ZOOKEEPER_HOME}/bin/zkServer.sh stop fi From 224db8ce0ff4e8befb772cde43870dd2ac7c080b Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Thu, 13 Oct 2022 15:18:16 +0800 Subject: [PATCH 2/3] ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have changed --- run.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run.sh b/run.sh index a9ef41ba22..b697f69e9d 100755 --- a/run.sh +++ b/run.sh @@ -492,10 +492,12 @@ function run_start_zk() type nc >/dev/null 2>&1 || { echo >&2 "start zk failed, need install netcat command..."; exit 1;} INSTALL_DIR=`pwd`/.zk_install - if [ -d "zookeeper-bin" ]; then - # this zookeeper-bin must have been got from github action workflows, thus just - # move it to ${INSTALL_DIR} to prevent from downloading - mv zookeeper-bin ${INSTALL_DIR}/ + if [ ! -d "${INSTALL_DIR}/zookeeper-bin" ]; then + if [ -d "zookeeper-bin" ]; then + # this zookeeper-bin must have been got from github action workflows, thus just + # move it to ${INSTALL_DIR} to prevent from downloading + mv zookeeper-bin ${INSTALL_DIR}/ + fi fi PORT=22181 while [[ $# > 0 ]]; do From fe79e3c837e9f5131f1d44c12d7a204abe1cc3d3 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 18 Oct 2022 18:13:21 +0800 Subject: [PATCH 3/3] ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have changed --- scripts/clear_zk.sh | 2 ++ scripts/start_zk.sh | 1 + scripts/stop_zk.sh | 2 ++ 3 files changed, 5 insertions(+) diff --git a/scripts/clear_zk.sh b/scripts/clear_zk.sh index 11267e73c8..311e05a95d 100755 --- a/scripts/clear_zk.sh +++ b/scripts/clear_zk.sh @@ -34,6 +34,8 @@ then fi cd $INSTALL_DIR + +# If the old dir for zk bin exists, just use it ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin if [ -d "${ZOOKEEPER_HOME}" ]; then diff --git a/scripts/start_zk.sh b/scripts/start_zk.sh index d23c1e89a9..2bc37356b0 100755 --- a/scripts/start_zk.sh +++ b/scripts/start_zk.sh @@ -44,6 +44,7 @@ fi cd "$INSTALL_DIR" || exit +# If the old dir for zk bin exists, just use it ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin if [ ! -d "${ZOOKEEPER_HOME}" ]; then diff --git a/scripts/stop_zk.sh b/scripts/stop_zk.sh index 0b4c45cbe2..c308680484 100755 --- a/scripts/stop_zk.sh +++ b/scripts/stop_zk.sh @@ -33,6 +33,8 @@ then fi cd $INSTALL_DIR + +# If the old dir for zk bin exists, just use it ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin if [ ! -d "${ZOOKEEPER_HOME}" ]; then