diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh
index 8b79455d0cd18..59f36c3c098aa 100755
--- a/docker/install/ubuntu_install_python_package.sh
+++ b/docker/install/ubuntu_install_python_package.sh
@@ -32,10 +32,12 @@ pip3 install \
     packaging \
     Pillow \
     pytest \
+    tlcpack-sphinx-addon==0.2.1 \
     pytest-profiling \
     pytest-xdist \
     requests \
     scipy \
     synr==0.6.0 \
     six \
+    ninja \
     tornado
diff --git a/tests/scripts/task_ci_setup.sh b/tests/scripts/task_ci_setup.sh
index 071de4859c686..01a7d80ce8dd8 100755
--- a/tests/scripts/task_ci_setup.sh
+++ b/tests/scripts/task_ci_setup.sh
@@ -30,8 +30,6 @@ set -o pipefail
 #
 echo "Additional setup in ${CI_IMAGE_NAME}"
 
-python3 -m pip install --user tlcpack-sphinx-addon==0.2.1 synr==0.6.0
-
 # Rebuild standalone_crt in build/ tree. This file is not currently archived by pack_lib() in
 # Jenkinsfile. We expect config.cmake to be present from pack_lib().
 # TODO(areusch): Make pack_lib() pack all the data dependencies of TVM.