From 06ea9344cc62a7ec81dbeac436d16be18e4d2c28 Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Fri, 22 Feb 2019 17:26:32 +0000 Subject: [PATCH 1/7] [DOCKER] Use curl -L consistently. Make the use of the curl -L option in docker build scripts consistent. --- docker/install/ubuntu_install_rust.sh | 2 +- docker/install/ubuntu_install_sgx.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/install/ubuntu_install_rust.sh b/docker/install/ubuntu_install_rust.sh index ab75802c84ed..c91dfdfb7909 100644 --- a/docker/install/ubuntu_install_rust.sh +++ b/docker/install/ubuntu_install_rust.sh @@ -3,7 +3,7 @@ apt-get update && apt-get install -y --no-install-recommends --force-yes curl export RUSTUP_HOME=/opt/rust export CARGO_HOME=/opt/rust # this rustc is one supported by the installed version of rust-sgx-sdk -curl -s -S https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28 +curl -s -S -L https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28 . $CARGO_HOME/env rustup toolchain add nightly rustup component add rust-src diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh index aea93d294d27..f24575eaacc3 100644 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -8,7 +8,7 @@ apt-get update && apt-get install -y --no-install-recommends --force-yes \ git clone https://github.com/intel/linux-sgx.git cd linux-sgx git checkout sgx_2.2 -curl -s -S 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply +curl -s -S -L 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply ./download_prebuilt.sh make -j4 sdk && make -j4 sdk_install_pkg ./linux/installer/bin/sgx_linux_x64_sdk*.bin --prefix /opt @@ -17,5 +17,5 @@ cd - git clone https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk cd /opt/rust-sgx-sdk git checkout 6098af # v1.0.5 -curl -s -S 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply +curl -s -S -L 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply cd - From ce4896029a648c5c23f6783df09b95d098223ead Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Fri, 22 Feb 2019 12:01:04 +0000 Subject: [PATCH 2/7] [DOCKER] Make all install .sh scripts directly executable. --- docker/install/install_tvm_cpu.sh | 2 ++ docker/install/install_tvm_gpu.sh | 2 ++ docker/install/ubuntu_install_androidsdk.sh | 2 ++ docker/install/ubuntu_install_antlr.sh | 2 ++ docker/install/ubuntu_install_caffe2.sh | 2 ++ docker/install/ubuntu_install_core.sh | 2 ++ docker/install/ubuntu_install_coreml.sh | 2 ++ docker/install/ubuntu_install_darknet.sh | 2 ++ docker/install/ubuntu_install_emscripten.sh | 2 ++ docker/install/ubuntu_install_gluoncv.sh | 2 ++ docker/install/ubuntu_install_golang.sh | 2 ++ docker/install/ubuntu_install_gradle.sh | 2 ++ docker/install/ubuntu_install_iverilog.sh | 2 ++ docker/install/ubuntu_install_java.sh | 2 ++ docker/install/ubuntu_install_keras.sh | 2 ++ docker/install/ubuntu_install_llvm.sh | 2 ++ docker/install/ubuntu_install_mxnet.sh | 2 ++ docker/install/ubuntu_install_nnpack.sh | 2 ++ docker/install/ubuntu_install_nodejs.sh | 2 ++ docker/install/ubuntu_install_onnx.sh | 2 ++ docker/install/ubuntu_install_opencl.sh | 2 ++ docker/install/ubuntu_install_opengl.sh | 4 +++- docker/install/ubuntu_install_python.sh | 2 ++ docker/install/ubuntu_install_python_package.sh | 2 ++ docker/install/ubuntu_install_redis.sh | 2 ++ docker/install/ubuntu_install_rocm.sh | 2 ++ docker/install/ubuntu_install_rust.sh | 2 ++ docker/install/ubuntu_install_sgx.sh | 2 ++ docker/install/ubuntu_install_sphinx.sh | 2 ++ docker/install/ubuntu_install_tensorflow.sh | 2 ++ docker/install/ubuntu_install_tflite.sh | 2 ++ docker/install/ubuntu_install_vulkan.sh | 2 +- 32 files changed, 64 insertions(+), 2 deletions(-) mode change 100644 => 100755 docker/install/install_tvm_cpu.sh mode change 100644 => 100755 docker/install/install_tvm_gpu.sh mode change 100644 => 100755 docker/install/ubuntu_install_androidsdk.sh mode change 100644 => 100755 docker/install/ubuntu_install_antlr.sh mode change 100644 => 100755 docker/install/ubuntu_install_caffe2.sh mode change 100644 => 100755 docker/install/ubuntu_install_core.sh mode change 100644 => 100755 docker/install/ubuntu_install_coreml.sh mode change 100644 => 100755 docker/install/ubuntu_install_darknet.sh mode change 100644 => 100755 docker/install/ubuntu_install_emscripten.sh mode change 100644 => 100755 docker/install/ubuntu_install_gluoncv.sh mode change 100644 => 100755 docker/install/ubuntu_install_golang.sh mode change 100644 => 100755 docker/install/ubuntu_install_gradle.sh mode change 100644 => 100755 docker/install/ubuntu_install_iverilog.sh mode change 100644 => 100755 docker/install/ubuntu_install_java.sh mode change 100644 => 100755 docker/install/ubuntu_install_keras.sh mode change 100644 => 100755 docker/install/ubuntu_install_llvm.sh mode change 100644 => 100755 docker/install/ubuntu_install_mxnet.sh mode change 100644 => 100755 docker/install/ubuntu_install_nnpack.sh mode change 100644 => 100755 docker/install/ubuntu_install_nodejs.sh mode change 100644 => 100755 docker/install/ubuntu_install_onnx.sh mode change 100644 => 100755 docker/install/ubuntu_install_opencl.sh mode change 100644 => 100755 docker/install/ubuntu_install_opengl.sh mode change 100644 => 100755 docker/install/ubuntu_install_python.sh mode change 100644 => 100755 docker/install/ubuntu_install_python_package.sh mode change 100644 => 100755 docker/install/ubuntu_install_redis.sh mode change 100644 => 100755 docker/install/ubuntu_install_rocm.sh mode change 100644 => 100755 docker/install/ubuntu_install_rust.sh mode change 100644 => 100755 docker/install/ubuntu_install_sgx.sh mode change 100644 => 100755 docker/install/ubuntu_install_sphinx.sh mode change 100644 => 100755 docker/install/ubuntu_install_tensorflow.sh mode change 100644 => 100755 docker/install/ubuntu_install_tflite.sh mode change 100644 => 100755 docker/install/ubuntu_install_vulkan.sh diff --git a/docker/install/install_tvm_cpu.sh b/docker/install/install_tvm_cpu.sh old mode 100644 new mode 100755 index 461ad244d37c..49a20684a347 --- a/docker/install/install_tvm_cpu.sh +++ b/docker/install/install_tvm_cpu.sh @@ -1,3 +1,5 @@ +#!/bin/bash + cd /usr git clone https://github.com/dmlc/tvm --recursive cd /usr/tvm diff --git a/docker/install/install_tvm_gpu.sh b/docker/install/install_tvm_gpu.sh old mode 100644 new mode 100755 index 8a1324646fd5..3afafce7389a --- a/docker/install/install_tvm_gpu.sh +++ b/docker/install/install_tvm_gpu.sh @@ -1,3 +1,5 @@ +#!/bin/bash + cd /usr git clone https://github.com/dmlc/tvm --recursive cd /usr/tvm diff --git a/docker/install/ubuntu_install_androidsdk.sh b/docker/install/ubuntu_install_androidsdk.sh old mode 100644 new mode 100755 index fa21d57c410f..f3c5e5d28f2e --- a/docker/install/ubuntu_install_androidsdk.sh +++ b/docker/install/ubuntu_install_androidsdk.sh @@ -1,3 +1,5 @@ +#!/bin/bash + . /etc/profile set -o errexit -o nounset diff --git a/docker/install/ubuntu_install_antlr.sh b/docker/install/ubuntu_install_antlr.sh old mode 100644 new mode 100755 index 6eb004213778..074e95a8106a --- a/docker/install/ubuntu_install_antlr.sh +++ b/docker/install/ubuntu_install_antlr.sh @@ -1,3 +1,5 @@ +#!/bin/bash + cd /usr/local/lib wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar cd - diff --git a/docker/install/ubuntu_install_caffe2.sh b/docker/install/ubuntu_install_caffe2.sh old mode 100644 new mode 100755 index 5fe827927e87..531eb05f8960 --- a/docker/install/ubuntu_install_caffe2.sh +++ b/docker/install/ubuntu_install_caffe2.sh @@ -1,3 +1,5 @@ +#!/bin/bash + python3 -m caffe2.python.models.download -i -f squeezenet python3 -m caffe2.python.models.download -i -f resnet50 python3 -m caffe2.python.models.download -i -f vgg19 diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh old mode 100644 new mode 100755 index efc69c946b97..1ab7b01e0d22 --- a/docker/install/ubuntu_install_core.sh +++ b/docker/install/ubuntu_install_core.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # install libraries for building c++ core on ubuntu apt-get update && apt-get install -y --no-install-recommends --force-yes \ git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev\ diff --git a/docker/install/ubuntu_install_coreml.sh b/docker/install/ubuntu_install_coreml.sh old mode 100644 new mode 100755 index 4b0fd126c61d..f47b107c49c8 --- a/docker/install/ubuntu_install_coreml.sh +++ b/docker/install/ubuntu_install_coreml.sh @@ -1 +1,3 @@ +#!/bin/bash + pip3 install coremltools diff --git a/docker/install/ubuntu_install_darknet.sh b/docker/install/ubuntu_install_darknet.sh old mode 100644 new mode 100755 index ecfb19626bc5..296f3bca8089 --- a/docker/install/ubuntu_install_darknet.sh +++ b/docker/install/ubuntu_install_darknet.sh @@ -1,3 +1,5 @@ +#!/bin/bash + #install the necessary dependancies, cffi, opencv wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so pip2 install opencv-python cffi diff --git a/docker/install/ubuntu_install_emscripten.sh b/docker/install/ubuntu_install_emscripten.sh old mode 100644 new mode 100755 index 4902538c9cfa..ce92e7d50ee4 --- a/docker/install/ubuntu_install_emscripten.sh +++ b/docker/install/ubuntu_install_emscripten.sh @@ -1,3 +1,5 @@ +#!/bin/bash + alias make="make -j4" # Get latest cmake diff --git a/docker/install/ubuntu_install_gluoncv.sh b/docker/install/ubuntu_install_gluoncv.sh old mode 100644 new mode 100755 index 0ca1a34cbc24..a3ec4c6bd097 --- a/docker/install/ubuntu_install_gluoncv.sh +++ b/docker/install/ubuntu_install_gluoncv.sh @@ -1 +1,3 @@ +#!/bin/bash + pip3 install gluoncv diff --git a/docker/install/ubuntu_install_golang.sh b/docker/install/ubuntu_install_golang.sh old mode 100644 new mode 100755 index 2361ccfbd2e4..7e0d8a96d22f --- a/docker/install/ubuntu_install_golang.sh +++ b/docker/install/ubuntu_install_golang.sh @@ -1,3 +1,5 @@ +#!/bin/bash + #install the necessary dependancies for golang build apt-get update && apt-get install -y golang-1.10-go apt-get update && apt-get install -y golang-1.10-doc diff --git a/docker/install/ubuntu_install_gradle.sh b/docker/install/ubuntu_install_gradle.sh old mode 100644 new mode 100755 index 9cc3a170e8ea..ec2baaaab561 --- a/docker/install/ubuntu_install_gradle.sh +++ b/docker/install/ubuntu_install_gradle.sh @@ -1,3 +1,5 @@ +#!/bin/bash + . /etc/profile set -o errexit -o nounset diff --git a/docker/install/ubuntu_install_iverilog.sh b/docker/install/ubuntu_install_iverilog.sh old mode 100644 new mode 100755 index 358bf9dc8376..33a91e889098 --- a/docker/install/ubuntu_install_iverilog.sh +++ b/docker/install/ubuntu_install_iverilog.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get install -y --no-install-recommends --force-yes make bison flex wget -q ftp://icarus.com/pub/eda/verilog/v10/verilog-10.1.tar.gz tar xf verilog-10.1.tar.gz diff --git a/docker/install/ubuntu_install_java.sh b/docker/install/ubuntu_install_java.sh old mode 100644 new mode 100755 index 462edc491627..fafe7896f1e9 --- a/docker/install/ubuntu_install_java.sh +++ b/docker/install/ubuntu_install_java.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -o errexit -o nounset apt-get update && apt-get install -y openjdk-8-jdk maven test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre" diff --git a/docker/install/ubuntu_install_keras.sh b/docker/install/ubuntu_install_keras.sh old mode 100644 new mode 100755 index 33bc38c80972..009edc26f71b --- a/docker/install/ubuntu_install_keras.sh +++ b/docker/install/ubuntu_install_keras.sh @@ -1,2 +1,4 @@ +#!/bin/bash + pip2 install keras tensorflow h5py pip3 install keras tensorflow h5py diff --git a/docker/install/ubuntu_install_llvm.sh b/docker/install/ubuntu_install_llvm.sh old mode 100644 new mode 100755 index 6a20fb227d38..a699e7876c1e --- a/docker/install/ubuntu_install_llvm.sh +++ b/docker/install/ubuntu_install_llvm.sh @@ -1,3 +1,5 @@ +#!/bin/bash + echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\ >> /etc/apt/sources.list.d/llvm.list echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\ diff --git a/docker/install/ubuntu_install_mxnet.sh b/docker/install/ubuntu_install_mxnet.sh old mode 100644 new mode 100755 index 0e7e9e3939a8..41ec1ce65795 --- a/docker/install/ubuntu_install_mxnet.sh +++ b/docker/install/ubuntu_install_mxnet.sh @@ -1 +1,3 @@ +#!/bin/bash + pip3 install mxnet diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh old mode 100644 new mode 100755 index 83225d4aa820..9655a319b8e5 --- a/docker/install/ubuntu_install_nnpack.sh +++ b/docker/install/ubuntu_install_nnpack.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake diff --git a/docker/install/ubuntu_install_nodejs.sh b/docker/install/ubuntu_install_nodejs.sh old mode 100644 new mode 100755 index fd43b4149af4..add4e057d73a --- a/docker/install/ubuntu_install_nodejs.sh +++ b/docker/install/ubuntu_install_nodejs.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get update && apt-get install -y curl curl -s -S -L https://deb.nodesource.com/setup_6.x | bash - apt-get update && apt-get install -y nodejs diff --git a/docker/install/ubuntu_install_onnx.sh b/docker/install/ubuntu_install_onnx.sh old mode 100644 new mode 100755 index 517ea77ab81e..cdb669d191bd --- a/docker/install/ubuntu_install_onnx.sh +++ b/docker/install/ubuntu_install_onnx.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # fix to certain version for now pip2 install onnx>=1.1.0 pip3 install onnx>=1.1.0 diff --git a/docker/install/ubuntu_install_opencl.sh b/docker/install/ubuntu_install_opencl.sh old mode 100644 new mode 100755 index ca4d1d04fd5c..2be6f0324381 --- a/docker/install/ubuntu_install_opencl.sh +++ b/docker/install/ubuntu_install_opencl.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Install OpenCL runtime in nvidia docker. apt-get update && apt-get install -y --no-install-recommends --force-yes \ ocl-icd-opencl-dev \ diff --git a/docker/install/ubuntu_install_opengl.sh b/docker/install/ubuntu_install_opengl.sh old mode 100644 new mode 100755 index f8be6e351581..6cb5e908b2c8 --- a/docker/install/ubuntu_install_opengl.sh +++ b/docker/install/ubuntu_install_opengl.sh @@ -1,4 +1,6 @@ +#!/bin/bash + apt-get update --fix-missing apt-get install -y --no-install-recommends --force-yes \ - libgl1-mesa-dev libglfw3-dev \ No newline at end of file + libgl1-mesa-dev libglfw3-dev diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh old mode 100644 new mode 100755 index ec30e77fb400..0527c2d9d802 --- a/docker/install/ubuntu_install_python.sh +++ b/docker/install/ubuntu_install_python.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # install python and pip, don't modify this, modify install_python_package.sh apt-get update && apt-get install -y python-dev diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh old mode 100644 new mode 100755 index 3e54271afa48..13a85468a8d0 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # install libraries for python package on ubuntu pip2 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typing antlr4-python2-runtime attrs pip3 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typed_ast pytest mypy orderedset antlr4-python3-runtime attrs diff --git a/docker/install/ubuntu_install_redis.sh b/docker/install/ubuntu_install_redis.sh old mode 100644 new mode 100755 index dfc9a3c381b6..6192e6e92e66 --- a/docker/install/ubuntu_install_redis.sh +++ b/docker/install/ubuntu_install_redis.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get update && apt-get install -y redis-server pip2 install xgboost psutil pip3 install xgboost psutil diff --git a/docker/install/ubuntu_install_rocm.sh b/docker/install/ubuntu_install_rocm.sh old mode 100644 new mode 100755 index d050c20078b8..9abc1379382d --- a/docker/install/ubuntu_install_rocm.sh +++ b/docker/install/ubuntu_install_rocm.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Install ROCm cross compilation toolchain. wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add - echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main > /etc/apt/sources.list.d/rocm.list diff --git a/docker/install/ubuntu_install_rust.sh b/docker/install/ubuntu_install_rust.sh old mode 100644 new mode 100755 index c91dfdfb7909..d44892751553 --- a/docker/install/ubuntu_install_rust.sh +++ b/docker/install/ubuntu_install_rust.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get update && apt-get install -y --no-install-recommends --force-yes curl export RUSTUP_HOME=/opt/rust diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh old mode 100644 new mode 100755 index f24575eaacc3..efc77964a9c1 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -1,3 +1,5 @@ +#!/bin/bash + apt-get update && apt-get install -y --no-install-recommends --force-yes \ build-essential git cmake \ wget python pkg-config software-properties-common \ diff --git a/docker/install/ubuntu_install_sphinx.sh b/docker/install/ubuntu_install_sphinx.sh old mode 100644 new mode 100755 index ba04c2e25e6f..f1f437074c95 --- a/docker/install/ubuntu_install_sphinx.sh +++ b/docker/install/ubuntu_install_sphinx.sh @@ -1 +1,3 @@ +#!/bin/bash + pip3 install sphinx sphinx-gallery sphinx_rtd_theme sphinx_autodoc_annotation matplotlib Image commonmark>=0.7.3 docutils>=0.11 diff --git a/docker/install/ubuntu_install_tensorflow.sh b/docker/install/ubuntu_install_tensorflow.sh old mode 100644 new mode 100755 index b773fcfb027b..e61ff1dc8c75 --- a/docker/install/ubuntu_install_tensorflow.sh +++ b/docker/install/ubuntu_install_tensorflow.sh @@ -1 +1,3 @@ +#!/bin/bash + pip3 install tensorflow diff --git a/docker/install/ubuntu_install_tflite.sh b/docker/install/ubuntu_install_tflite.sh old mode 100644 new mode 100755 index 5df01f186c26..247b697401f0 --- a/docker/install/ubuntu_install_tflite.sh +++ b/docker/install/ubuntu_install_tflite.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Download, build and install flatbuffers git clone --recursive https://github.com/google/flatbuffers.git cd flatbuffers diff --git a/docker/install/ubuntu_install_vulkan.sh b/docker/install/ubuntu_install_vulkan.sh old mode 100644 new mode 100755 index 72a6139905e6..857f86467b18 --- a/docker/install/ubuntu_install_vulkan.sh +++ b/docker/install/ubuntu_install_vulkan.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash wget -q https://sdk.lunarg.com/sdk/download/1.0.65.0/linux/vulkansdk-linux-x86_64-1.0.65.0.run From 148229a9df7db427b8fe61ec5259f5e7386efa52 Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Fri, 22 Feb 2019 17:23:56 +0000 Subject: [PATCH 3/7] [DOCKER] Robustness -e -u -o pipefail The install scripts used to construct docker environments do not, in general, propagate errors. Some of the scripts use adhoc && directives to chain together short sequences of commands but there are numerous failure modes which are silently ignored. This patch puts in place some consistent, basic, shell error trapping across all of the install scripts. Note this is a step forward towards more robust scripts but it is not a complete solution. --- docker/install/install_tvm_cpu.sh | 4 ++++ docker/install/install_tvm_gpu.sh | 4 ++++ docker/install/ubuntu_install_androidsdk.sh | 3 ++- docker/install/ubuntu_install_antlr.sh | 4 ++++ docker/install/ubuntu_install_caffe2.sh | 4 ++++ docker/install/ubuntu_install_core.sh | 4 ++++ docker/install/ubuntu_install_coreml.sh | 4 ++++ docker/install/ubuntu_install_darknet.sh | 4 ++++ docker/install/ubuntu_install_emscripten.sh | 4 ++++ docker/install/ubuntu_install_gluoncv.sh | 4 ++++ docker/install/ubuntu_install_golang.sh | 4 ++++ docker/install/ubuntu_install_gradle.sh | 1 + docker/install/ubuntu_install_iverilog.sh | 4 ++++ docker/install/ubuntu_install_java.sh | 2 ++ docker/install/ubuntu_install_keras.sh | 4 ++++ docker/install/ubuntu_install_llvm.sh | 4 ++++ docker/install/ubuntu_install_mxnet.sh | 4 ++++ docker/install/ubuntu_install_nnpack.sh | 5 ++++- docker/install/ubuntu_install_nodejs.sh | 4 ++++ docker/install/ubuntu_install_onnx.sh | 4 ++++ docker/install/ubuntu_install_opencl.sh | 4 ++++ docker/install/ubuntu_install_opengl.sh | 4 ++++ docker/install/ubuntu_install_python.sh | 4 ++++ docker/install/ubuntu_install_python_package.sh | 4 ++++ docker/install/ubuntu_install_redis.sh | 4 ++++ docker/install/ubuntu_install_rocm.sh | 4 ++++ docker/install/ubuntu_install_rust.sh | 4 ++++ docker/install/ubuntu_install_sgx.sh | 4 ++++ docker/install/ubuntu_install_sphinx.sh | 4 ++++ docker/install/ubuntu_install_tensorflow.sh | 4 ++++ docker/install/ubuntu_install_tflite.sh | 4 ++++ docker/install/ubuntu_install_vulkan.sh | 4 ++++ 32 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docker/install/install_tvm_cpu.sh b/docker/install/install_tvm_cpu.sh index 49a20684a347..eed660365e9e 100755 --- a/docker/install/install_tvm_cpu.sh +++ b/docker/install/install_tvm_cpu.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + cd /usr git clone https://github.com/dmlc/tvm --recursive cd /usr/tvm diff --git a/docker/install/install_tvm_gpu.sh b/docker/install/install_tvm_gpu.sh index 3afafce7389a..c3d6ab47d2db 100755 --- a/docker/install/install_tvm_gpu.sh +++ b/docker/install/install_tvm_gpu.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + cd /usr git clone https://github.com/dmlc/tvm --recursive cd /usr/tvm diff --git a/docker/install/ubuntu_install_androidsdk.sh b/docker/install/ubuntu_install_androidsdk.sh index f3c5e5d28f2e..96fdbe168d6d 100755 --- a/docker/install/ubuntu_install_androidsdk.sh +++ b/docker/install/ubuntu_install_androidsdk.sh @@ -3,6 +3,7 @@ . /etc/profile set -o errexit -o nounset +set -o pipefail ANDROID_HOME=/opt/android-sdk-linux ASDKTOOLS_HOME=/opt/android-sdk-tools @@ -60,7 +61,7 @@ EOF mkdir /root/.android 2>/dev/null || true touch /root/.android/repositories.cfg -yes | sdkmanager --licenses --sdk_root="$ANDROID_HOME" +(yes || true) | sdkmanager --licenses --sdk_root="$ANDROID_HOME" sdkmanager --verbose --package_file=/install/package-list-minimal.txt --sdk_root="$ANDROID_HOME" test -d "${ANDROID_HOME}/build-tools/27.0.3" test -d "${ANDROID_HOME}/ndk-bundle" diff --git a/docker/install/ubuntu_install_antlr.sh b/docker/install/ubuntu_install_antlr.sh index 074e95a8106a..6dae3ae12d56 100755 --- a/docker/install/ubuntu_install_antlr.sh +++ b/docker/install/ubuntu_install_antlr.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + cd /usr/local/lib wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar cd - diff --git a/docker/install/ubuntu_install_caffe2.sh b/docker/install/ubuntu_install_caffe2.sh index 531eb05f8960..bb9322704918 100755 --- a/docker/install/ubuntu_install_caffe2.sh +++ b/docker/install/ubuntu_install_caffe2.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + python3 -m caffe2.python.models.download -i -f squeezenet python3 -m caffe2.python.models.download -i -f resnet50 python3 -m caffe2.python.models.download -i -f vgg19 diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh index 1ab7b01e0d22..ce744deed42e 100755 --- a/docker/install/ubuntu_install_core.sh +++ b/docker/install/ubuntu_install_core.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # install libraries for building c++ core on ubuntu apt-get update && apt-get install -y --no-install-recommends --force-yes \ git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev\ diff --git a/docker/install/ubuntu_install_coreml.sh b/docker/install/ubuntu_install_coreml.sh index f47b107c49c8..51afc1423961 100755 --- a/docker/install/ubuntu_install_coreml.sh +++ b/docker/install/ubuntu_install_coreml.sh @@ -1,3 +1,7 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip3 install coremltools diff --git a/docker/install/ubuntu_install_darknet.sh b/docker/install/ubuntu_install_darknet.sh index 296f3bca8089..5c350b848bf7 100755 --- a/docker/install/ubuntu_install_darknet.sh +++ b/docker/install/ubuntu_install_darknet.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + #install the necessary dependancies, cffi, opencv wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so pip2 install opencv-python cffi diff --git a/docker/install/ubuntu_install_emscripten.sh b/docker/install/ubuntu_install_emscripten.sh index ce92e7d50ee4..4671c898438a 100755 --- a/docker/install/ubuntu_install_emscripten.sh +++ b/docker/install/ubuntu_install_emscripten.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + alias make="make -j4" # Get latest cmake diff --git a/docker/install/ubuntu_install_gluoncv.sh b/docker/install/ubuntu_install_gluoncv.sh index a3ec4c6bd097..adfbdce7c7b1 100755 --- a/docker/install/ubuntu_install_gluoncv.sh +++ b/docker/install/ubuntu_install_gluoncv.sh @@ -1,3 +1,7 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip3 install gluoncv diff --git a/docker/install/ubuntu_install_golang.sh b/docker/install/ubuntu_install_golang.sh index 7e0d8a96d22f..e9470eb00546 100755 --- a/docker/install/ubuntu_install_golang.sh +++ b/docker/install/ubuntu_install_golang.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + #install the necessary dependancies for golang build apt-get update && apt-get install -y golang-1.10-go apt-get update && apt-get install -y golang-1.10-doc diff --git a/docker/install/ubuntu_install_gradle.sh b/docker/install/ubuntu_install_gradle.sh index ec2baaaab561..7f62406ca710 100755 --- a/docker/install/ubuntu_install_gradle.sh +++ b/docker/install/ubuntu_install_gradle.sh @@ -3,6 +3,7 @@ . /etc/profile set -o errexit -o nounset +set -o pipefail GRADLE_HOME=/opt/gradle GRADLE_VERSION=4.10-rc-2 diff --git a/docker/install/ubuntu_install_iverilog.sh b/docker/install/ubuntu_install_iverilog.sh index 33a91e889098..24c240d92c69 100755 --- a/docker/install/ubuntu_install_iverilog.sh +++ b/docker/install/ubuntu_install_iverilog.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get install -y --no-install-recommends --force-yes make bison flex wget -q ftp://icarus.com/pub/eda/verilog/v10/verilog-10.1.tar.gz tar xf verilog-10.1.tar.gz diff --git a/docker/install/ubuntu_install_java.sh b/docker/install/ubuntu_install_java.sh index fafe7896f1e9..e1f431bee845 100755 --- a/docker/install/ubuntu_install_java.sh +++ b/docker/install/ubuntu_install_java.sh @@ -1,6 +1,8 @@ #!/bin/bash set -o errexit -o nounset +set -o pipefail + apt-get update && apt-get install -y openjdk-8-jdk maven test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre" echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /etc/profile diff --git a/docker/install/ubuntu_install_keras.sh b/docker/install/ubuntu_install_keras.sh index 009edc26f71b..b689949d0dff 100755 --- a/docker/install/ubuntu_install_keras.sh +++ b/docker/install/ubuntu_install_keras.sh @@ -1,4 +1,8 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip2 install keras tensorflow h5py pip3 install keras tensorflow h5py diff --git a/docker/install/ubuntu_install_llvm.sh b/docker/install/ubuntu_install_llvm.sh index a699e7876c1e..91094f764be8 100755 --- a/docker/install/ubuntu_install_llvm.sh +++ b/docker/install/ubuntu_install_llvm.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\ >> /etc/apt/sources.list.d/llvm.list echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\ diff --git a/docker/install/ubuntu_install_mxnet.sh b/docker/install/ubuntu_install_mxnet.sh index 41ec1ce65795..a15dca7def07 100755 --- a/docker/install/ubuntu_install_mxnet.sh +++ b/docker/install/ubuntu_install_mxnet.sh @@ -1,3 +1,7 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip3 install mxnet diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh index 9655a319b8e5..e224e6e059b6 100755 --- a/docker/install/ubuntu_install_nnpack.sh +++ b/docker/install/ubuntu_install_nnpack.sh @@ -1,7 +1,10 @@ #!/bin/bash -apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake +set -e +set -u +set -o pipefail +apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake git clone https://github.com/Maratyszcza/NNPACK NNPACK cd NNPACK diff --git a/docker/install/ubuntu_install_nodejs.sh b/docker/install/ubuntu_install_nodejs.sh index add4e057d73a..850f88bcdefa 100755 --- a/docker/install/ubuntu_install_nodejs.sh +++ b/docker/install/ubuntu_install_nodejs.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get update && apt-get install -y curl curl -s -S -L https://deb.nodesource.com/setup_6.x | bash - apt-get update && apt-get install -y nodejs diff --git a/docker/install/ubuntu_install_onnx.sh b/docker/install/ubuntu_install_onnx.sh index cdb669d191bd..2778a2489667 100755 --- a/docker/install/ubuntu_install_onnx.sh +++ b/docker/install/ubuntu_install_onnx.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # fix to certain version for now pip2 install onnx>=1.1.0 pip3 install onnx>=1.1.0 diff --git a/docker/install/ubuntu_install_opencl.sh b/docker/install/ubuntu_install_opencl.sh index 2be6f0324381..ee9d8b87955a 100755 --- a/docker/install/ubuntu_install_opencl.sh +++ b/docker/install/ubuntu_install_opencl.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # Install OpenCL runtime in nvidia docker. apt-get update && apt-get install -y --no-install-recommends --force-yes \ ocl-icd-opencl-dev \ diff --git a/docker/install/ubuntu_install_opengl.sh b/docker/install/ubuntu_install_opengl.sh index 6cb5e908b2c8..eb79260cfe21 100755 --- a/docker/install/ubuntu_install_opengl.sh +++ b/docker/install/ubuntu_install_opengl.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get update --fix-missing apt-get install -y --no-install-recommends --force-yes \ diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh index 0527c2d9d802..1001488bf1f4 100755 --- a/docker/install/ubuntu_install_python.sh +++ b/docker/install/ubuntu_install_python.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # install python and pip, don't modify this, modify install_python_package.sh apt-get update && apt-get install -y python-dev diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index 13a85468a8d0..c15ff75f260e 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # install libraries for python package on ubuntu pip2 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typing antlr4-python2-runtime attrs pip3 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typed_ast pytest mypy orderedset antlr4-python3-runtime attrs diff --git a/docker/install/ubuntu_install_redis.sh b/docker/install/ubuntu_install_redis.sh index 6192e6e92e66..d079170b0536 100755 --- a/docker/install/ubuntu_install_redis.sh +++ b/docker/install/ubuntu_install_redis.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get update && apt-get install -y redis-server pip2 install xgboost psutil pip3 install xgboost psutil diff --git a/docker/install/ubuntu_install_rocm.sh b/docker/install/ubuntu_install_rocm.sh index 9abc1379382d..b7fab7b1401f 100755 --- a/docker/install/ubuntu_install_rocm.sh +++ b/docker/install/ubuntu_install_rocm.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # Install ROCm cross compilation toolchain. wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add - echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main > /etc/apt/sources.list.d/rocm.list diff --git a/docker/install/ubuntu_install_rust.sh b/docker/install/ubuntu_install_rust.sh index d44892751553..f037c7d835d1 100755 --- a/docker/install/ubuntu_install_rust.sh +++ b/docker/install/ubuntu_install_rust.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get update && apt-get install -y --no-install-recommends --force-yes curl export RUSTUP_HOME=/opt/rust diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh index efc77964a9c1..93b4f571ce31 100755 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + apt-get update && apt-get install -y --no-install-recommends --force-yes \ build-essential git cmake \ wget python pkg-config software-properties-common \ diff --git a/docker/install/ubuntu_install_sphinx.sh b/docker/install/ubuntu_install_sphinx.sh index f1f437074c95..50e1e92796c3 100755 --- a/docker/install/ubuntu_install_sphinx.sh +++ b/docker/install/ubuntu_install_sphinx.sh @@ -1,3 +1,7 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip3 install sphinx sphinx-gallery sphinx_rtd_theme sphinx_autodoc_annotation matplotlib Image commonmark>=0.7.3 docutils>=0.11 diff --git a/docker/install/ubuntu_install_tensorflow.sh b/docker/install/ubuntu_install_tensorflow.sh index e61ff1dc8c75..4fdf9c0d46ab 100755 --- a/docker/install/ubuntu_install_tensorflow.sh +++ b/docker/install/ubuntu_install_tensorflow.sh @@ -1,3 +1,7 @@ #!/bin/bash +set -e +set -u +set -o pipefail + pip3 install tensorflow diff --git a/docker/install/ubuntu_install_tflite.sh b/docker/install/ubuntu_install_tflite.sh index 247b697401f0..9b9c19aa63f7 100755 --- a/docker/install/ubuntu_install_tflite.sh +++ b/docker/install/ubuntu_install_tflite.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + # Download, build and install flatbuffers git clone --recursive https://github.com/google/flatbuffers.git cd flatbuffers diff --git a/docker/install/ubuntu_install_vulkan.sh b/docker/install/ubuntu_install_vulkan.sh index 857f86467b18..6772b029cc90 100755 --- a/docker/install/ubuntu_install_vulkan.sh +++ b/docker/install/ubuntu_install_vulkan.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -u +set -o pipefail + wget -q https://sdk.lunarg.com/sdk/download/1.0.65.0/linux/vulkansdk-linux-x86_64-1.0.65.0.run bash vulkansdk-linux-x86_64-1.0.65.0.run From 9e3f360743b6b73e92621f836b5c25246148c687 Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Fri, 22 Feb 2019 16:58:14 +0000 Subject: [PATCH 4/7] [DOCKER] Drop use of --force-yes The --force-yes option is generally not recommend, it can leave systems in an undefined state. The use of --allow-* options is preferred. In this particular case the --force-yes option appears to serve no purpose. Dropping it. --- docker/install/ubuntu_install_core.sh | 2 +- docker/install/ubuntu_install_iverilog.sh | 2 +- docker/install/ubuntu_install_llvm.sh | 2 +- docker/install/ubuntu_install_nnpack.sh | 2 +- docker/install/ubuntu_install_opencl.sh | 2 +- docker/install/ubuntu_install_opengl.sh | 2 +- docker/install/ubuntu_install_rocm.sh | 2 +- docker/install/ubuntu_install_rust.sh | 2 +- docker/install/ubuntu_install_sgx.sh | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh index ce744deed42e..c7e2918971fd 100755 --- a/docker/install/ubuntu_install_core.sh +++ b/docker/install/ubuntu_install_core.sh @@ -5,7 +5,7 @@ set -u set -o pipefail # install libraries for building c++ core on ubuntu -apt-get update && apt-get install -y --no-install-recommends --force-yes \ +apt-get update && apt-get install -y --no-install-recommends \ git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev\ libcurl4-openssl-dev libopenblas-dev g++ sudo diff --git a/docker/install/ubuntu_install_iverilog.sh b/docker/install/ubuntu_install_iverilog.sh index 24c240d92c69..2304f697affd 100755 --- a/docker/install/ubuntu_install_iverilog.sh +++ b/docker/install/ubuntu_install_iverilog.sh @@ -4,7 +4,7 @@ set -e set -u set -o pipefail -apt-get install -y --no-install-recommends --force-yes make bison flex +apt-get install -y --no-install-recommends make bison flex wget -q ftp://icarus.com/pub/eda/verilog/v10/verilog-10.1.tar.gz tar xf verilog-10.1.tar.gz cd verilog-10.1 diff --git a/docker/install/ubuntu_install_llvm.sh b/docker/install/ubuntu_install_llvm.sh index 91094f764be8..a562c3258628 100755 --- a/docker/install/ubuntu_install_llvm.sh +++ b/docker/install/ubuntu_install_llvm.sh @@ -25,4 +25,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main\ >> /etc/apt/sources.list.d/llvm.list wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - -apt-get update && apt-get install -y --force-yes llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0 +apt-get update && apt-get install -y llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0 diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh index e224e6e059b6..f146918cf1ae 100755 --- a/docker/install/ubuntu_install_nnpack.sh +++ b/docker/install/ubuntu_install_nnpack.sh @@ -4,7 +4,7 @@ set -e set -u set -o pipefail -apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake +apt-get update && apt-get install -y --no-install-recommends git cmake git clone https://github.com/Maratyszcza/NNPACK NNPACK cd NNPACK diff --git a/docker/install/ubuntu_install_opencl.sh b/docker/install/ubuntu_install_opencl.sh index ee9d8b87955a..f16de615c4b1 100755 --- a/docker/install/ubuntu_install_opencl.sh +++ b/docker/install/ubuntu_install_opencl.sh @@ -5,7 +5,7 @@ set -u set -o pipefail # Install OpenCL runtime in nvidia docker. -apt-get update && apt-get install -y --no-install-recommends --force-yes \ +apt-get update && apt-get install -y --no-install-recommends \ ocl-icd-opencl-dev \ clinfo && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/install/ubuntu_install_opengl.sh b/docker/install/ubuntu_install_opengl.sh index eb79260cfe21..82050c14f307 100755 --- a/docker/install/ubuntu_install_opengl.sh +++ b/docker/install/ubuntu_install_opengl.sh @@ -6,5 +6,5 @@ set -o pipefail apt-get update --fix-missing -apt-get install -y --no-install-recommends --force-yes \ +apt-get install -y --no-install-recommends \ libgl1-mesa-dev libglfw3-dev diff --git a/docker/install/ubuntu_install_rocm.sh b/docker/install/ubuntu_install_rocm.sh index b7fab7b1401f..be7f2364bf63 100755 --- a/docker/install/ubuntu_install_rocm.sh +++ b/docker/install/ubuntu_install_rocm.sh @@ -7,4 +7,4 @@ set -o pipefail # Install ROCm cross compilation toolchain. wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add - echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main > /etc/apt/sources.list.d/rocm.list -apt-get update && apt-get install -y --force-yes rocm-dev +apt-get update && apt-get install -y rocm-dev diff --git a/docker/install/ubuntu_install_rust.sh b/docker/install/ubuntu_install_rust.sh index f037c7d835d1..67bcd15cbc84 100755 --- a/docker/install/ubuntu_install_rust.sh +++ b/docker/install/ubuntu_install_rust.sh @@ -4,7 +4,7 @@ set -e set -u set -o pipefail -apt-get update && apt-get install -y --no-install-recommends --force-yes curl +apt-get update && apt-get install -y --no-install-recommends curl export RUSTUP_HOME=/opt/rust export CARGO_HOME=/opt/rust diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh index 93b4f571ce31..79ca23fe4055 100755 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -4,7 +4,7 @@ set -e set -u set -o pipefail -apt-get update && apt-get install -y --no-install-recommends --force-yes \ +apt-get update && apt-get install -y --no-install-recommends \ build-essential git cmake \ wget python pkg-config software-properties-common \ autoconf automake libtool ocaml \ From ef579faad021a3ef311cf95533b18138aacddd5a Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Fri, 22 Feb 2019 17:01:27 +0000 Subject: [PATCH 5/7] [DOCKER] Drop superflous repeated apt-get update. The "apt-get update && apt-get install" idiom is necessary and specific to Dockerfile. In shell the repeated apt-get update is superflous. Drop the duplicates. --- docker/install/ubuntu_install_golang.sh | 7 ++++--- docker/install/ubuntu_install_nodejs.sh | 10 ++++++++-- docker/install/ubuntu_install_python.sh | 11 +++++++---- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docker/install/ubuntu_install_golang.sh b/docker/install/ubuntu_install_golang.sh index e9470eb00546..c29e764cbb3a 100755 --- a/docker/install/ubuntu_install_golang.sh +++ b/docker/install/ubuntu_install_golang.sh @@ -5,6 +5,7 @@ set -u set -o pipefail #install the necessary dependancies for golang build -apt-get update && apt-get install -y golang-1.10-go -apt-get update && apt-get install -y golang-1.10-doc -apt-get update && apt-get install -y golint +apt-get update +apt-get install -y golang-1.10-go +apt-get install -y golang-1.10-doc +apt-get install -y golint diff --git a/docker/install/ubuntu_install_nodejs.sh b/docker/install/ubuntu_install_nodejs.sh index 850f88bcdefa..dfdd0432e4db 100755 --- a/docker/install/ubuntu_install_nodejs.sh +++ b/docker/install/ubuntu_install_nodejs.sh @@ -4,7 +4,13 @@ set -e set -u set -o pipefail -apt-get update && apt-get install -y curl +apt-get update +apt-get install -y curl + +# The node install script fetched and executed here will update the +# apt source list, hence the second apt-get update is necessary. curl -s -S -L https://deb.nodesource.com/setup_6.x | bash - -apt-get update && apt-get install -y nodejs +apt-get update +apt-get install -y nodejs + npm install eslint jsdoc ws diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh index 1001488bf1f4..43c27b1b2def 100755 --- a/docker/install/ubuntu_install_python.sh +++ b/docker/install/ubuntu_install_python.sh @@ -5,12 +5,15 @@ set -u set -o pipefail # install python and pip, don't modify this, modify install_python_package.sh -apt-get update && apt-get install -y python-dev +apt-get update +apt-get install -y python-dev # python 3.6 -apt-get update && apt-get install -y software-properties-common -add-apt-repository ppa:jonathonf/python-3.6 &&\ - apt-get update && apt-get install -y python-pip python-dev python3.6 python3.6-dev +apt-get install -y software-properties-common + +add-apt-repository ppa:jonathonf/python-3.6 +apt-get update +apt-get install -y python-pip python-dev python3.6 python3.6-dev rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 From 0dd4c1f27bc8221850d1783b2d871d0e8fdd9322 Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Thu, 28 Feb 2019 17:22:32 +0000 Subject: [PATCH 6/7] [DOCKER] Use clone --branch rather than clone then checkout Use the git clone --branch idiom rather than git clone && git checkout. This paves the way for using --depth=1 --- docker/install/ubuntu_install_nnpack.sh | 5 +---- docker/install/ubuntu_install_sgx.sh | 7 +++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh index f146918cf1ae..ad88cffd3588 100755 --- a/docker/install/ubuntu_install_nnpack.sh +++ b/docker/install/ubuntu_install_nnpack.sh @@ -6,11 +6,8 @@ set -o pipefail apt-get update && apt-get install -y --no-install-recommends git cmake -git clone https://github.com/Maratyszcza/NNPACK NNPACK -cd NNPACK # TODO: specific tag? -git checkout 1e005b0c2 -cd - +git clone --branch=1e005b0c2 https://github.com/Maratyszcza/NNPACK NNPACK mkdir -p NNPACK/build cd NNPACK/build diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh index 79ca23fe4055..e6c7624a636d 100755 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -11,17 +11,16 @@ apt-get update && apt-get install -y --no-install-recommends \ protobuf-compiler libprotobuf-dev \ libssl-dev libcurl4-openssl-dev curl -git clone https://github.com/intel/linux-sgx.git +git clone --branch=sgx_2.2 https://github.com/intel/linux-sgx.git cd linux-sgx -git checkout sgx_2.2 curl -s -S -L 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply ./download_prebuilt.sh make -j4 sdk && make -j4 sdk_install_pkg ./linux/installer/bin/sgx_linux_x64_sdk*.bin --prefix /opt cd - -git clone https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk +tag=6098af # v1.0.5 +git clone --branch=$tag https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk cd /opt/rust-sgx-sdk -git checkout 6098af # v1.0.5 curl -s -S -L 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply cd - From fdd0539db9b5bdc857d7ec8779d93f307d9f0271 Mon Sep 17 00:00:00 2001 From: Marcus Shawcroft Date: Tue, 26 Feb 2019 11:27:53 +0000 Subject: [PATCH 7/7] [DOCKER] Shallow clone. Use shallow clone to reduce bandwidth requirements of repeated docker (re)-builds. --- docker/Dockerfile.ci_gpu | 2 +- docker/Dockerfile.demo_opencl | 2 +- docker/install/install_tvm_cpu.sh | 2 +- docker/install/install_tvm_gpu.sh | 2 +- docker/install/ubuntu_install_nnpack.sh | 2 +- docker/install/ubuntu_install_sgx.sh | 4 ++-- docker/install/ubuntu_install_tflite.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu index 6a599b1e3917..a83d7000d0fe 100644 --- a/docker/Dockerfile.ci_gpu +++ b/docker/Dockerfile.ci_gpu @@ -24,7 +24,7 @@ COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh RUN bash /install/ubuntu_install_sphinx.sh # Fix recommonmark to latest version -RUN git clone https://github.com/rtfd/recommonmark +RUN git clone --depth=1 https://github.com/rtfd/recommonmark RUN cd recommonmark; python3 setup.py install # Enable doxygen for c++ doc build diff --git a/docker/Dockerfile.demo_opencl b/docker/Dockerfile.demo_opencl index 460b901bf08f..2d0b45983902 100644 --- a/docker/Dockerfile.demo_opencl +++ b/docker/Dockerfile.demo_opencl @@ -45,7 +45,7 @@ RUN echo "Cloning TVM source & submodules" ENV TVM_PAR_DIR="/usr" RUN mkdir -p TVM_PAR_DIR && \ cd ${TVM_PAR_DIR} && \ - git clone https://github.com/dmlc/tvm --recursive + git clone --depth=1 https://github.com/dmlc/tvm --recursive #RUN git submodule update --init --recursive diff --git a/docker/install/install_tvm_cpu.sh b/docker/install/install_tvm_cpu.sh index eed660365e9e..04153559d27e 100755 --- a/docker/install/install_tvm_cpu.sh +++ b/docker/install/install_tvm_cpu.sh @@ -5,7 +5,7 @@ set -u set -o pipefail cd /usr -git clone https://github.com/dmlc/tvm --recursive +git clone --depth=1 https://github.com/dmlc/tvm --recursive cd /usr/tvm echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake echo set\(USE_RPC ON\) >> config.cmake diff --git a/docker/install/install_tvm_gpu.sh b/docker/install/install_tvm_gpu.sh index c3d6ab47d2db..d31e10ce9ab9 100755 --- a/docker/install/install_tvm_gpu.sh +++ b/docker/install/install_tvm_gpu.sh @@ -5,7 +5,7 @@ set -u set -o pipefail cd /usr -git clone https://github.com/dmlc/tvm --recursive +git clone --depth=1 https://github.com/dmlc/tvm --recursive cd /usr/tvm echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake echo set\(USE_CUDA ON\) >> config.cmake diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh index ad88cffd3588..1cf044a9b257 100755 --- a/docker/install/ubuntu_install_nnpack.sh +++ b/docker/install/ubuntu_install_nnpack.sh @@ -7,7 +7,7 @@ set -o pipefail apt-get update && apt-get install -y --no-install-recommends git cmake # TODO: specific tag? -git clone --branch=1e005b0c2 https://github.com/Maratyszcza/NNPACK NNPACK +git clone --branch=1e005b0c2 --depth=1 https://github.com/Maratyszcza/NNPACK NNPACK mkdir -p NNPACK/build cd NNPACK/build diff --git a/docker/install/ubuntu_install_sgx.sh b/docker/install/ubuntu_install_sgx.sh index e6c7624a636d..d2958e5d0893 100755 --- a/docker/install/ubuntu_install_sgx.sh +++ b/docker/install/ubuntu_install_sgx.sh @@ -11,7 +11,7 @@ apt-get update && apt-get install -y --no-install-recommends \ protobuf-compiler libprotobuf-dev \ libssl-dev libcurl4-openssl-dev curl -git clone --branch=sgx_2.2 https://github.com/intel/linux-sgx.git +git clone --branch=sgx_2.2 --depth=1 https://github.com/intel/linux-sgx.git cd linux-sgx curl -s -S -L 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply ./download_prebuilt.sh @@ -20,7 +20,7 @@ make -j4 sdk && make -j4 sdk_install_pkg cd - tag=6098af # v1.0.5 -git clone --branch=$tag https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk +git clone --branch=$tag --depth=1 https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk cd /opt/rust-sgx-sdk curl -s -S -L 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply cd - diff --git a/docker/install/ubuntu_install_tflite.sh b/docker/install/ubuntu_install_tflite.sh index 9b9c19aa63f7..ed8ea1deff3f 100755 --- a/docker/install/ubuntu_install_tflite.sh +++ b/docker/install/ubuntu_install_tflite.sh @@ -5,7 +5,7 @@ set -u set -o pipefail # Download, build and install flatbuffers -git clone --recursive https://github.com/google/flatbuffers.git +git clone --depth=1 --recursive https://github.com/google/flatbuffers.git cd flatbuffers cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release make install -j8