build(images): download apache-zookeeper-3.7.0-bin.tar.gz during third-parties image building for unit tests

.github/workflows/lint_and_test_cpp.yaml

@@ -93,7 +93,7 @@ jobs:
           mkdir build
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
           cmake --build build/ -j $(nproc)
-          ./scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_hadoop.sh hadoop-bin
       - name: Compilation
         run: |
           ccache -p
@@ -209,7 +209,7 @@ jobs:
           mkdir build
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
           cmake --build build/ -j $(nproc)
-          ./scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_hadoop.sh hadoop-bin
       - name: Compilation
         run: |
           ccache -p
@@ -321,7 +321,7 @@ jobs:
           mkdir build
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
           cmake --build build/ -j $(nproc)
-          ./scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_hadoop.sh hadoop-bin
       - name: Compilation
         run: |
           ccache -p
@@ -433,7 +433,7 @@ jobs:
           mkdir build
           cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
           cmake --build build/ -j $(nproc)
-          ./scripts/download_hadoop.sh hadoop-bin
+          ../scripts/download_hadoop.sh hadoop-bin
       - name: Compilation
         run: |
           ccache -p

.github/workflows/thirdparty-regular-push.yml

@@ -171,6 +171,7 @@ jobs:
             OS_VERSION=${{ matrix.osversion }}
             ROCKSDB_PORTABLE=ON
             HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
@@ -207,5 +208,6 @@ jobs:
             ROCKSDB_PORTABLE=ON
             USE_JEMALLOC=ON
             HADOOP_BIN_PATH=hadoop-bin
+            ZOOKEEPER_BIN_PATH=zookeeper-bin
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

docker/thirdparties-bin/Dockerfile

@@ -29,12 +29,14 @@ ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
 ARG ROCKSDB_PORTABLE=OFF
 ARG USE_JEMALLOC=OFF
 ARG HADOOP_BIN_PATH
+ARG ZOOKEEPER_BIN_PATH
 RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
     && cd incubator-pegasus/thirdparty \
     && unzip /root/thirdparties-src.zip -d . \
     && cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ . \
     && cmake --build build/ -j $(($(nproc)/2+1)) \
     && ../scripts/download_hadoop.sh ${HADOOP_BIN_PATH} \
-    && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} \
+    && ../scripts/download_zk.sh ${ZOOKEEPER_BIN_PATH} \
+    && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} ${ZOOKEEPER_BIN_PATH} \
     && cd ~ \
     && rm -rf incubator-pegasus;

scripts/download_hadoop.sh

@@ -19,50 +19,13 @@
 
 set -e
 
-if [ $# -lt 1 ]; then
-    echo "Target HADOOP_BIN_PATH is not provided, thus do not try to download hadoop"
-    exit 0
-fi
+CWD=$(cd $(dirname $0) && pwd)
 
-HADOOP_BIN_PATH=$1
-if [ -d ${HADOOP_BIN_PATH} ]; then
-    echo "Target HADOOP_BIN_PATH ${HADOOP_BIN_PATH} has existed, thus do not try to download hadoop"
-    exit 0
+if [ $# -ge 1 ]; then
+    HADOOP_BIN_PATH=$1
 fi
 
 HADOOP_VERSION=2.8.4
 HADOOP_DIR_NAME=hadoop-${HADOOP_VERSION}
-HADOOP_PACKAGE_NAME=${HADOOP_DIR_NAME}.tar.gz
-if [ ! -f ${HADOOP_PACKAGE_NAME} ]; then
-    echo "Downloading hadoop..."
-
-    DOWNLOAD_URL="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/${HADOOP_PACKAGE_NAME}"
-    if ! wget -T 10 -t 5 ${DOWNLOAD_URL}; then
-        echo "ERROR: download hadoop failed"
-        exit 1
-    fi
-
-    HADOOP_PACKAGE_MD5="b30b409bb69185003b3babd1504ba224"
-    if [ `md5sum ${HADOOP_PACKAGE_NAME} | awk '{print$1}'` != ${HADOOP_PACKAGE_MD5} ]; then
-        echo "Check file ${HADOOP_PACKAGE_NAME} md5sum failed!"
-        exit 1
-    fi
-fi
-
-rm -rf ${HADOOP_DIR_NAME}
-
-echo "Decompressing hadoop..."
-if ! tar xf ${HADOOP_PACKAGE_NAME}; then
-    echo "ERROR: decompress hadoop failed"
-    rm -f ${HADOOP_PACKAGE_NAME}
-    exit 1
-fi
-
-rm -f ${HADOOP_PACKAGE_NAME}
-
-if [ ! -d ${HADOOP_DIR_NAME} ]; then
-    echo "ERROR: ${HADOOP_DIR_NAME} does not exist"
-    exit 1
-fi
-
-mv ${HADOOP_DIR_NAME} ${HADOOP_BIN_PATH}
+HADOOP_PACKAGE_MD5="b30b409bb69185003b3babd1504ba224"
+${CWD}/download_package.sh ${HADOOP_DIR_NAME} ${HADOOP_PACKAGE_MD5} ${HADOOP_BIN_PATH}

scripts/download_package.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "Invalid arguments !"
+    echo "USAGE: $0 <DIR_NAME> <PACKAGE_MD5> [TARGET_PATH]"
+    exit 1
+fi
+
+DIR_NAME=$1
+PACKAGE_MD5=$2
+
+if [ $# -lt 3 ]; then
+    echo "TARGET_PATH is not provided, thus do not try to download ${DIR_NAME}"
+    exit 0
+fi
+
+TARGET_PATH=$3
+if [ -d ${TARGET_PATH} ]; then
+    echo "TARGET_PATH ${TARGET_PATH} has existed, thus do not try to download ${DIR_NAME}"
+    exit 0
+fi
+
+PACKAGE_NAME=${DIR_NAME}.tar.gz
+if [ ! -f ${PACKAGE_NAME} ]; then
+    echo "Downloading ${DIR_NAME}..."
+
+    DOWNLOAD_URL="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/${PACKAGE_NAME}"
+    if ! wget -T 10 -t 5 ${DOWNLOAD_URL}; then
+        echo "ERROR: download ${DIR_NAME} failed"
+        exit 1
+    fi
+
+    if [ `md5sum ${PACKAGE_NAME} | awk '{print$1}'` != ${PACKAGE_MD5} ]; then
+        echo "Check file ${PACKAGE_NAME} md5sum failed!"
+        exit 1
+    fi
+fi
+
+rm -rf ${DIR_NAME}
+
+echo "Decompressing ${DIR_NAME}..."
+if ! tar xf ${PACKAGE_NAME}; then
+    echo "ERROR: decompress ${DIR_NAME} failed"
+    rm -f ${PACKAGE_NAME}
+    exit 1
+fi
+
+rm -f ${PACKAGE_NAME}
+
+if [ ! -d ${DIR_NAME} ]; then
+    echo "ERROR: ${DIR_NAME} does not exist"
+    exit 1
+fi
+
+mv ${DIR_NAME} ${TARGET_PATH}

scripts/download_zk.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+CWD=$(cd $(dirname $0) && pwd)
+
+if [ $# -ge 1 ]; then
+    ZOOKEEPER_BIN_PATH=$1
+fi
+
+ZOOKEEPER_VERSION=3.7.0
+ZOOKEEPER_DIR_NAME=apache-zookeeper-${ZOOKEEPER_VERSION}-bin
+ZOOKEEPER_PACKAGE_MD5="8ffa97e7e6b0b2cf1d022e5156a7561a"
+${CWD}/download_package.sh ${ZOOKEEPER_DIR_NAME} ${ZOOKEEPER_PACKAGE_MD5} ${ZOOKEEPER_BIN_PATH}