diff --git a/.github/workflows/template-setup-e2e-test/action.yaml b/.github/workflows/template-setup-e2e-test/action.yaml
index f9cbd08b106..ed7af516e6f 100644
--- a/.github/workflows/template-setup-e2e-test/action.yaml
+++ b/.github/workflows/template-setup-e2e-test/action.yaml
@@ -29,4 +29,4 @@ runs:
 
     - name: Install Katib SDK
       shell: bash
-      run: pip install -e sdk/python/v1beta1
+      run: pip install --prefer-binary -e sdk/python/v1beta1
diff --git a/Makefile b/Makefile
index 6b8a665f3ad..5c66e19fa6e 100755
--- a/Makefile
+++ b/Makefile
@@ -42,7 +42,7 @@ endif
 
 yamllint:
 ifndef HAS_YAMLLINT
-	pip install yamllint
+	pip install --prefer-binary yamllint
 	$(info "yamllint has been installed")
 endif
 	hack/verify-yamllint.sh
@@ -150,16 +150,16 @@ update-boilerplate:
 	./hack/boilerplate/update-boilerplate.sh
 
 prepare-pytest:
-	pip install -r test/unit/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
-	pip install -r cmd/suggestion/pbt/v1beta1/requirements.txt
-	pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
-	pip install -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
+	pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/optuna/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/hyperband/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/pbt/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
 
 prepare-pytest-testdata:
 ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
diff --git a/cmd/earlystopping/medianstop/v1beta1/Dockerfile b/cmd/earlystopping/medianstop/v1beta1/Dockerfile
index a32fbb06169..22c8a02ffc1 100644
--- a/cmd/earlystopping/medianstop/v1beta1/Dockerfile
+++ b/cmd/earlystopping/medianstop/v1beta1/Dockerfile
@@ -17,7 +17,7 @@ ADD ./${EARLY_STOPPING_DIR}/ ${TARGET_DIR}/${EARLY_STOPPING_DIR}/
 
 WORKDIR  ${TARGET_DIR}/${EARLY_STOPPING_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
index 9bb78fcdd75..4624cf0a507 100644
--- a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
+++ b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
@@ -17,7 +17,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le
index fbc819dce37..ff6c84bd532 100644
--- a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le
+++ b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le
@@ -1,6 +1,6 @@
 FROM ibmcom/tensorflow-ppc64le:2.2.0-py3
 ADD . /usr/src/app/github.com/kubeflow/katib
 WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/apis/manager/v1beta1/python:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/tfevent-metricscollector/:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/common/
 ENTRYPOINT ["python", "main.py"]
diff --git a/cmd/suggestion/hyperband/v1beta1/Dockerfile b/cmd/suggestion/hyperband/v1beta1/Dockerfile
index 2afceb05e9f..6b8bd88f1ef 100644
--- a/cmd/suggestion/hyperband/v1beta1/Dockerfile
+++ b/cmd/suggestion/hyperband/v1beta1/Dockerfile
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/hyperopt/v1beta1/Dockerfile b/cmd/suggestion/hyperopt/v1beta1/Dockerfile
index 9c7bc2e6bf2..81b1a77d2f4 100644
--- a/cmd/suggestion/hyperopt/v1beta1/Dockerfile
+++ b/cmd/suggestion/hyperopt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperopt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/nas/darts/v1beta1/Dockerfile b/cmd/suggestion/nas/darts/v1beta1/Dockerfile
index 9f26214a3ca..b06ed57180d 100644
--- a/cmd/suggestion/nas/darts/v1beta1/Dockerfile
+++ b/cmd/suggestion/nas/darts/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/nas/darts/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/nas/enas/v1beta1/Dockerfile b/cmd/suggestion/nas/enas/v1beta1/Dockerfile
index 8b12354ee32..4c2eb0d64ae 100644
--- a/cmd/suggestion/nas/enas/v1beta1/Dockerfile
+++ b/cmd/suggestion/nas/enas/v1beta1/Dockerfile
@@ -14,7 +14,7 @@ ENV SUGGESTION_DIR cmd/suggestion/nas/enas/v1beta1
 ENV GRPC_HEALTH_PROBE_VERSION v0.4.11
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -27,7 +27,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/optuna/v1beta1/Dockerfile b/cmd/suggestion/optuna/v1beta1/Dockerfile
index 9105b54d5dc..33c665283ed 100644
--- a/cmd/suggestion/optuna/v1beta1/Dockerfile
+++ b/cmd/suggestion/optuna/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/optuna/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/pbt/v1beta1/Dockerfile b/cmd/suggestion/pbt/v1beta1/Dockerfile
index 795cdf009cb..02bb195f735 100644
--- a/cmd/suggestion/pbt/v1beta1/Dockerfile
+++ b/cmd/suggestion/pbt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/pbt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/cmd/suggestion/skopt/v1beta1/Dockerfile b/cmd/suggestion/skopt/v1beta1/Dockerfile
index c6d71b534b9..539c72e5011 100644
--- a/cmd/suggestion/skopt/v1beta1/Dockerfile
+++ b/cmd/suggestion/skopt/v1beta1/Dockerfile
@@ -13,7 +13,7 @@ ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/skopt/v1beta1
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
-RUN if [ "${TARGETARCH}" = "ppc64le" ] || [ "${TARGETARCH}" = "aarch64" ]; then \
+RUN if [ "${TARGETARCH}" = "ppc64le" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
     apt-get clean && \
@@ -26,7 +26,7 @@ COPY --from=downloader /bin/grpc_health_probe /bin/grpc_health_probe
 
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
index 7437e1867bd..08ddaa1c4a7 100644
--- a/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
+++ b/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
@@ -6,7 +6,7 @@ ADD examples/v1beta1/trial-images/darts-cnn-cifar10 ${TARGET_DIR}
 
 WORKDIR  ${TARGET_DIR}
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
index b5933d60a81..0e5b5aeb5de 100644
--- a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
+++ b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
@@ -15,7 +15,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
index 39d8d305e6c..770f964078c 100644
--- a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
+++ b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
@@ -10,7 +10,7 @@ ADD examples/v1beta1/trial-images/enas-cnn-cifar10 ${TARGET_DIR}
 
 WORKDIR  ${TARGET_DIR}
 
-RUN pip install --no-cache-dir scipy==1.8.1
+RUN pip install --prefer-binary --no-cache-dir scipy==1.8.1
 RUN chgrp -R 0 ${TARGET_DIR} \
   && chmod -R g+rwX ${TARGET_DIR}
 
diff --git a/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile b/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
index e18359bceb9..f10c8c75e77 100644
--- a/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
+++ b/examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
@@ -20,7 +20,7 @@ RUN apt-get -y update \
 RUN if [ "${TARGETARCH}" = "arm64" ]; then \
       /opt/mxnet-mnist/install-arm-performance-libraries.sh; \
     fi
-RUN pip install -r requirements.txt
+RUN pip install --prefer-binary -r requirements.txt
 RUN chgrp -R 0 /opt/mxnet-mnist \
   && chmod -R g+rwX /opt/mxnet-mnist
 
diff --git a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
index 5fef0094386..f1641889fe8 100644
--- a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
+++ b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
@@ -6,7 +6,7 @@ WORKDIR /opt/pytorch-mnist
 
 # Add folder for the logs.
 RUN mkdir /katib
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 /opt/pytorch-mnist \
   && chmod -R g+rwX /opt/pytorch-mnist \
diff --git a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu
index af1b7de3b43..0216c351606 100644
--- a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu
+++ b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu
@@ -9,7 +9,7 @@ WORKDIR /opt/pytorch-mnist
 
 # Add folder for the logs.
 RUN mkdir /katib
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 /opt/pytorch-mnist \
   && chmod -R g+rwX /opt/pytorch-mnist \
diff --git a/examples/v1beta1/trial-images/simple-pbt/Dockerfile b/examples/v1beta1/trial-images/simple-pbt/Dockerfile
index b05de3d35e3..cf4833b111c 100644
--- a/examples/v1beta1/trial-images/simple-pbt/Dockerfile
+++ b/examples/v1beta1/trial-images/simple-pbt/Dockerfile
@@ -4,7 +4,7 @@ ADD examples/v1beta1/trial-images/simple-pbt /opt/pbt
 
 WORKDIR /opt/pbt
 
-RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install --prefer-binary -r requirements.txt
 RUN chgrp -R 0 /opt/pbt \
   && chmod -R g+rwX /opt/pbt
 
diff --git a/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile b/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
index f589dd4608f..781a44d8d8b 100644
--- a/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
+++ b/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
@@ -13,7 +13,7 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \
     rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --prefer-binary --no-cache-dir -r requirements.txt
 RUN chgrp -R 0 /opt/tf-mnist-with-summaries \
   && chmod -R g+rwX /opt/tf-mnist-with-summaries
 
diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py
index 2a709c55002..97c46772611 100644
--- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py
+++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py
@@ -102,7 +102,7 @@ def get_script_for_python_packages(packages_to_install, pip_index_url):
             python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
         fi
 
-        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \
+        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --prefer-binary --quiet \
         --no-warn-script-location --index-url {pip_index_url} {packages_str}
         """
     )