From c29ea4a5876aecfceb9211c4eed79d266b55dc53 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Thu, 7 May 2020 23:54:55 +0000 Subject: [PATCH 1/5] Android build containers --- ci/build.py | 5 +- ...android_armv7 => Dockerfile.build.android} | 56 +++++++++++++---- ci/docker/Dockerfile.build.android_armv8 | 63 ------------------- ci/docker/docker-compose.yml | 19 ++++++ 4 files changed, 65 insertions(+), 78 deletions(-) rename ci/docker/{Dockerfile.build.android_armv7 => Dockerfile.build.android} (59%) delete mode 100644 ci/docker/Dockerfile.build.android_armv8 diff --git a/ci/build.py b/ci/build.py index 0f23c3519311..ca72a95989e7 100755 --- a/ci/build.py +++ b/ci/build.py @@ -46,9 +46,10 @@ DOCKER_COMPOSE_WHITELIST = ('centos7_cpu', 'centos7_gpu_cu92', 'centos7_gpu_cu100', 'centos7_gpu_cu101', 'centos7_gpu_cu102', 'ubuntu_cpu', 'ubuntu_build_cuda', 'ubuntu_gpu_cu101', 'publish.test.centos7_cpu', - 'publish.test.centos7_gpu') + 'publish.test.centos7_gpu', 'android_armv7', 'android_armv8') # Files for docker compose -DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/publish.test.centos7')) +DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/build.android', + 'docker/publish.test.centos7')) def get_dockerfiles_path(): diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android similarity index 59% rename from ci/docker/Dockerfile.build.android_armv7 rename to ci/docker/Dockerfile.build.android index 8d9fb6481e2e..67bb3c4c71e8 100644 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android @@ -16,14 +16,12 @@ # specific language governing permissions and limitations # under the License. # -# Dockerfile to build MXNet for Android ARMv7 +# Dockerfile to build MXNet for Android -FROM ubuntu:20.04 - -ENV ARCH=armv7l \ - HOSTCC=gcc \ - HOSTCXX=g++ \ - TARGET=ARMV7 +#################################################################################################### +# Shared base for all Android targets +#################################################################################################### +FROM ubuntu:20.04 AS base WORKDIR /usr/local @@ -42,6 +40,24 @@ RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/re rm android-ndk-r19-linux-x86_64.zip ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ +WORKDIR /work/build + + +#################################################################################################### +# Specialize base image for ARMv7 +#################################################################################################### +FROM base as armv7 +ENV ARCH=armv7l \ + HOSTCC=gcc \ + HOSTCXX=g++ \ + TARGET=ARMV7 + RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ mkdir /usr/local/openblas-android && \ cd /usr/local/OpenBLAS && \ @@ -54,10 +70,24 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ rm -rf OpenBLAS ENV OpenBLAS_HOME=/usr/local/openblas-android -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh -COPY runtime_functions.sh /work/ -WORKDIR /work/build +#################################################################################################### +# Specialize base image for ARMv8 +#################################################################################################### +FROM base as armv8 +ENV ARCH=aarch64 \ + HOSTCC=gcc \ + HOSTCXX=g++ \ + TARGET=ARMV8 + +RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ + mkdir /usr/local/openblas-android && \ + cd /usr/local/OpenBLAS && \ + export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \ + make NOFORTRAN=1 NO_SHARED=1 \ + LDFLAGS="-L/usr/local/android-ndk-r21/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" \ + CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang AR=$TOOLCHAIN/bin/aarch64-linux-android-ar && \ + make PREFIX=/usr/local/openblas-android NO_SHARED=1 install && \ + cd /usr/local && \ + rm -rf OpenBLAS +ENV OpenBLAS_HOME=/usr/local/openblas-android diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 deleted file mode 100644 index a78113a33bae..000000000000 --- a/ci/docker/Dockerfile.build.android_armv8 +++ /dev/null @@ -1,63 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build MXNet for Android ARM64/ARMv8 - -FROM ubuntu:20.04 - -ENV ARCH=aarch64 \ - HOSTCC=gcc \ - HOSTCXX=g++ \ - TARGET=ARMV8 - -WORKDIR /usr/local - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - build-essential \ - ninja-build \ - cmake \ - ccache \ - git \ - curl \ - unzip \ - && rm -rf /var/lib/apt/lists/* - -RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip && \ - unzip android-ndk-r19-linux-x86_64.zip && \ - rm android-ndk-r19-linux-x86_64.zip -ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake - -RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ - mkdir /usr/local/openblas-android && \ - cd /usr/local/OpenBLAS && \ - export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \ - make NOFORTRAN=1 NO_SHARED=1 \ - LDFLAGS="-L/usr/local/android-ndk-r21/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" \ - CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang AR=$TOOLCHAIN/bin/aarch64-linux-android-ar && \ - make PREFIX=/usr/local/openblas-android NO_SHARED=1 install && \ - cd /usr/local && \ - rm -rf OpenBLAS -ENV OpenBLAS_HOME=/usr/local/openblas-android - -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ -WORKDIR /work/build diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index 5ac660df796b..9fd75177ba97 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -119,6 +119,25 @@ services: cache_from: - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_build_cuda:latest ################################################################################################### + # Dockerfile.build.android based images used for testing cross-compilation for Android + ################################################################################################### + build.android_armv7: + image: ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest + build: + context: . + dockerfile: Dockerfile.build.android + target: armv7 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest + build.android_armv8: + image: ${DOCKER_CACHE_REGISTRY}/build.android_armv8:latest + build: + context: . + dockerfile: Dockerfile.build.android + target: armv8 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/build.android_armv8:latest + ################################################################################################### # Dockerfile.publish.test based images used for testing binary artifacts on minimal systems. ################################################################################################### publish.test.centos7_cpu: From e394312f09833c1bcec9c81eeae778df86a3ef2d Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 8 May 2020 00:05:04 +0000 Subject: [PATCH 2/5] ARM build containers --- ci/build.py | 5 +- ...rfile.build.armv6 => Dockerfile.build.arm} | 85 ++++++++++++++++--- ci/docker/Dockerfile.build.armv7 | 59 ------------- ci/docker/Dockerfile.build.armv8 | 59 ------------- ci/docker/docker-compose.yml | 27 ++++++ 5 files changed, 102 insertions(+), 133 deletions(-) rename ci/docker/{Dockerfile.build.armv6 => Dockerfile.build.arm} (50%) delete mode 100644 ci/docker/Dockerfile.build.armv7 delete mode 100644 ci/docker/Dockerfile.build.armv8 diff --git a/ci/build.py b/ci/build.py index ca72a95989e7..842b95353793 100755 --- a/ci/build.py +++ b/ci/build.py @@ -46,10 +46,11 @@ DOCKER_COMPOSE_WHITELIST = ('centos7_cpu', 'centos7_gpu_cu92', 'centos7_gpu_cu100', 'centos7_gpu_cu101', 'centos7_gpu_cu102', 'ubuntu_cpu', 'ubuntu_build_cuda', 'ubuntu_gpu_cu101', 'publish.test.centos7_cpu', - 'publish.test.centos7_gpu', 'android_armv7', 'android_armv8') + 'publish.test.centos7_gpu', 'android_armv7', 'android_armv8', + 'armv6', 'armv7', 'armv8') # Files for docker compose DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/build.android', - 'docker/publish.test.centos7')) + 'docker/build.arm', 'docker/publish.test.centos7')) def get_dockerfiles_path(): diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.arm similarity index 50% rename from ci/docker/Dockerfile.build.armv6 rename to ci/docker/Dockerfile.build.arm index 83186369d829..d4b1daad3635 100644 --- a/ci/docker/Dockerfile.build.armv6 +++ b/ci/docker/Dockerfile.build.arm @@ -16,14 +16,12 @@ # specific language governing permissions and limitations # under the License. # -# Dockerfile to build MXNet for ARMv6 +# Dockerfile to build MXNet for ARM -FROM ubuntu:20.04 - -ENV ARCH=armv6l \ - HOSTCC=gcc \ - HOSTCXX=g++ \ - TARGET=ARMV6 +#################################################################################################### +# Shared base for all ARM targets +#################################################################################################### +FROM ubuntu:20.04 AS base WORKDIR /usr/local @@ -39,6 +37,25 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ python3-pip \ && rm -rf /var/lib/apt/lists/* +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ +WORKDIR /work/mxnet + + +#################################################################################################### +# Specialize base image for ARMv6 +#################################################################################################### +FROM base as armv6 + +ENV ARCH=armv6l \ + HOSTCC=gcc \ + HOSTCXX=g++ \ + TARGET=ARMV6 + # We use a toolchain from toolchains.bootlin.com instead of Debian / Ubunut # crossbuild-essential-armel toolchain, as the latter targets ARM architecture # versions 4T, 5T, and 6, whereas we only wish to target ARMV6 and like to use @@ -55,10 +72,52 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ cd /usr/local && \ rm -rf OpenBLAS -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh -COPY runtime_functions.sh /work/ -WORKDIR /work/mxnet +#################################################################################################### +# Specialize base image for ARMv7 +#################################################################################################### +FROM base as armv7 + +ENV ARCH=armv7l \ + HOSTCC=gcc \ + HOSTCXX=g++ \ + TARGET=ARMV7 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + crossbuild-essential-armhf \ + && rm -rf /var/lib/apt/lists/* + +COPY toolchains/arm-linux-gnueabihf-toolchain.cmake /usr/local +ENV CMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf-toolchain.cmake + +RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ + cd /usr/local/OpenBLAS && \ + make NOFORTRAN=1 NO_SHARED=1 CC=arm-linux-gnueabihf-gcc && \ + make PREFIX=/usr/local/arm-linux-gnueabihf NO_SHARED=1 install && \ + cd /usr/local && \ + rm -rf OpenBLAS + + +#################################################################################################### +# Specialize base image for ARMv8 +#################################################################################################### +FROM base as armv8 + +ENV ARCH=aarch64 \ + HOSTCC=gcc \ + HOSTCXX=g++ \ + TARGET=ARMV8 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + crossbuild-essential-arm64 \ + && rm -rf /var/lib/apt/lists/* + +COPY toolchains/aarch64-linux-gnu-toolchain.cmake /usr +ENV CMAKE_TOOLCHAIN_FILE=/usr/aarch64-linux-gnu-toolchain.cmake + +RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ + cd /usr/local/OpenBLAS && \ + make NOFORTRAN=1 NO_SHARED=1 CC=aarch64-linux-gnu-gcc && \ + make PREFIX=/usr/aarch64-linux-gnu NO_SHARED=1 install && \ + cd /usr/local && \ + rm -rf OpenBLAS diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 deleted file mode 100644 index d207d79485ae..000000000000 --- a/ci/docker/Dockerfile.build.armv7 +++ /dev/null @@ -1,59 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build MXNet for ARMv7 - -FROM ubuntu:20.04 - -ENV ARCH=armv7l \ - HOSTCC=gcc \ - HOSTCXX=g++ \ - TARGET=ARMV7 - -WORKDIR /usr/local - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - build-essential \ - ninja-build \ - cmake \ - ccache \ - git \ - curl \ - zip \ - python3 \ - python3-pip \ - crossbuild-essential-armhf \ - && rm -rf /var/lib/apt/lists/* - -COPY toolchains/arm-linux-gnueabihf-toolchain.cmake /usr/local -ENV CMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf-toolchain.cmake - -RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ - cd /usr/local/OpenBLAS && \ - make NOFORTRAN=1 NO_SHARED=1 CC=arm-linux-gnueabihf-gcc && \ - make PREFIX=/usr/local/arm-linux-gnueabihf NO_SHARED=1 install && \ - cd /usr/local && \ - rm -rf OpenBLAS - -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ -WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 deleted file mode 100644 index d318cc2f02d4..000000000000 --- a/ci/docker/Dockerfile.build.armv8 +++ /dev/null @@ -1,59 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to build MXNet for ARM64/ARMv8 - -FROM ubuntu:20.04 - -ENV ARCH=aarch64 \ - HOSTCC=gcc \ - HOSTCXX=g++ \ - TARGET=ARMV8 - -WORKDIR /usr/local - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - build-essential \ - ninja-build \ - cmake \ - ccache \ - git \ - curl \ - zip \ - python3 \ - python3-pip \ - crossbuild-essential-arm64 \ - && rm -rf /var/lib/apt/lists/* - -COPY toolchains/aarch64-linux-gnu-toolchain.cmake /usr -ENV CMAKE_TOOLCHAIN_FILE=/usr/aarch64-linux-gnu-toolchain.cmake - -RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ - cd /usr/local/OpenBLAS && \ - make NOFORTRAN=1 NO_SHARED=1 CC=aarch64-linux-gnu-gcc && \ - make PREFIX=/usr/aarch64-linux-gnu NO_SHARED=1 install && \ - cd /usr/local && \ - rm -rf OpenBLAS - -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ -WORKDIR /work/mxnet diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index 9fd75177ba97..2da1626e7983 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -119,6 +119,33 @@ services: cache_from: - ${DOCKER_CACHE_REGISTRY}/build.ubuntu_build_cuda:latest ################################################################################################### + # Dockerfile.build.android based images used for testing cross-compilation for plain ARM + ################################################################################################### + build.armv6: + image: ${DOCKER_CACHE_REGISTRY}/build.armv6:latest + build: + context: . + dockerfile: Dockerfile.build.arm + target: armv6 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/build.armv6:latest + build.armv7: + image: ${DOCKER_CACHE_REGISTRY}/build.armv7:latest + build: + context: . + dockerfile: Dockerfile.build.arm + target: armv7 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/build.armv7:latest + build.armv8: + image: ${DOCKER_CACHE_REGISTRY}/build.armv8:latest + build: + context: . + dockerfile: Dockerfile.build.arm + target: armv8 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/build.armv8:latest + ################################################################################################### # Dockerfile.build.android based images used for testing cross-compilation for Android ################################################################################################### build.android_armv7: From 87d3ab82f5d0c1b2ffdcc549052bd71f2ce4da21 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 8 May 2020 00:09:01 +0000 Subject: [PATCH 3/5] ARM test containers --- ci/build.py | 4 +- ci/docker/Dockerfile.build.test.armv8 | 48 ------------------- ...e.build.test.armv7 => Dockerfile.test.arm} | 5 +- ci/docker/docker-compose.yml | 21 ++++++++ 4 files changed, 26 insertions(+), 52 deletions(-) delete mode 100644 ci/docker/Dockerfile.build.test.armv8 rename ci/docker/{Dockerfile.build.test.armv7 => Dockerfile.test.arm} (94%) diff --git a/ci/build.py b/ci/build.py index 842b95353793..55523389de7d 100755 --- a/ci/build.py +++ b/ci/build.py @@ -47,10 +47,10 @@ 'centos7_gpu_cu101', 'centos7_gpu_cu102', 'ubuntu_cpu', 'ubuntu_build_cuda', 'ubuntu_gpu_cu101', 'publish.test.centos7_cpu', 'publish.test.centos7_gpu', 'android_armv7', 'android_armv8', - 'armv6', 'armv7', 'armv8') + 'armv6', 'armv7', 'armv8', 'test.armv7', 'test.armv8') # Files for docker compose DOCKER_COMPOSE_FILES = set(('docker/build.centos7', 'docker/build.ubuntu', 'docker/build.android', - 'docker/build.arm', 'docker/publish.test.centos7')) + 'docker/build.arm', 'docker/test.arm', 'docker/publish.test.centos7')) def get_dockerfiles_path(): diff --git a/ci/docker/Dockerfile.build.test.armv8 b/ci/docker/Dockerfile.build.test.armv8 deleted file mode 100644 index 7a77c78bbeea..000000000000 --- a/ci/docker/Dockerfile.build.test.armv8 +++ /dev/null @@ -1,48 +0,0 @@ -# -*- mode: dockerfile -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Dockerfile to test MXNet on Ubuntu 20.04 ARMv8 CPU - -FROM arm64v8/ubuntu:20.04 - -WORKDIR /usr/local - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - python3 \ - python3-pip \ - python3-numpy \ - python3-scipy \ - python3-requests \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --no-cache-dir --upgrade pip && \ - pip3 install --no-cache-dir \ - pytest==5.3.5 \ - pytest-env==0.6.2 \ - pytest-cov==2.8.1 \ - pytest-xdist==1.31.0 \ - pytest-timeout==1.3.4 \ - mock==2.0.0 - -ARG USER_ID=0 -ARG GROUP_ID=0 -COPY install/ubuntu_adduser.sh /work/ -RUN /work/ubuntu_adduser.sh - -COPY runtime_functions.sh /work/ -WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.test.armv7 b/ci/docker/Dockerfile.test.arm similarity index 94% rename from ci/docker/Dockerfile.build.test.armv7 rename to ci/docker/Dockerfile.test.arm index 066040c5be8f..c02f4b73e422 100644 --- a/ci/docker/Dockerfile.build.test.armv7 +++ b/ci/docker/Dockerfile.test.arm @@ -16,9 +16,10 @@ # specific language governing permissions and limitations # under the License. # -# Dockerfile to test MXNet on Ubuntu 20.04 ARMv7 CPU +# Dockerfile to test MXNet on Ubuntu 20.04 ARM -FROM arm32v7/ubuntu:20.04 +ARG BASE_IMAGE +FROM $BASE_IMAGE WORKDIR /usr/local diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index 2da1626e7983..e8b2e550e473 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -146,6 +146,27 @@ services: cache_from: - ${DOCKER_CACHE_REGISTRY}/build.armv8:latest ################################################################################################### + # Dockerfile.test.arm based images for testing ARM artefacts via QEMU + ################################################################################################### + test.armv7: + image: ${DOCKER_CACHE_REGISTRY}/test.armv7:latest + build: + context: . + dockerfile: Dockerfile.test.arm + args: + BASE_IMAGE: arm32v7/ubuntu:20.04 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/test.armv7:latest + test.armv8: + image: ${DOCKER_CACHE_REGISTRY}/test.armv8:latest + build: + context: . + dockerfile: Dockerfile.test.arm + args: + BASE_IMAGE: arm64v8/ubuntu:20.04 + cache_from: + - ${DOCKER_CACHE_REGISTRY}/test.armv8:latest + ################################################################################################### # Dockerfile.build.android based images used for testing cross-compilation for Android ################################################################################################### build.android_armv7: From 4aaa8501e2b17d91428e40089ba0bd43b9a4426e Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 8 May 2020 17:55:54 +0000 Subject: [PATCH 4/5] Fix naming scheme --- ci/docker/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml index e8b2e550e473..b0fd89856c38 100644 --- a/ci/docker/docker-compose.yml +++ b/ci/docker/docker-compose.yml @@ -121,7 +121,7 @@ services: ################################################################################################### # Dockerfile.build.android based images used for testing cross-compilation for plain ARM ################################################################################################### - build.armv6: + armv6: image: ${DOCKER_CACHE_REGISTRY}/build.armv6:latest build: context: . @@ -129,7 +129,7 @@ services: target: armv6 cache_from: - ${DOCKER_CACHE_REGISTRY}/build.armv6:latest - build.armv7: + armv7: image: ${DOCKER_CACHE_REGISTRY}/build.armv7:latest build: context: . @@ -137,7 +137,7 @@ services: target: armv7 cache_from: - ${DOCKER_CACHE_REGISTRY}/build.armv7:latest - build.armv8: + armv8: image: ${DOCKER_CACHE_REGISTRY}/build.armv8:latest build: context: . @@ -169,7 +169,7 @@ services: ################################################################################################### # Dockerfile.build.android based images used for testing cross-compilation for Android ################################################################################################### - build.android_armv7: + android_armv7: image: ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest build: context: . @@ -177,7 +177,7 @@ services: target: armv7 cache_from: - ${DOCKER_CACHE_REGISTRY}/build.android_armv7:latest - build.android_armv8: + android_armv8: image: ${DOCKER_CACHE_REGISTRY}/build.android_armv8:latest build: context: . From 3599c01e8639a06955988182f338a629faf0e88f Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 8 May 2020 19:40:10 +0000 Subject: [PATCH 5/5] Set WORKDIR at correct location --- ci/docker/Dockerfile.build.android | 5 ++++- ci/docker/Dockerfile.build.arm | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/docker/Dockerfile.build.android b/ci/docker/Dockerfile.build.android index 67bb3c4c71e8..939e6b1fe45b 100644 --- a/ci/docker/Dockerfile.build.android +++ b/ci/docker/Dockerfile.build.android @@ -46,7 +46,6 @@ COPY install/ubuntu_adduser.sh /work/ RUN /work/ubuntu_adduser.sh COPY runtime_functions.sh /work/ -WORKDIR /work/build #################################################################################################### @@ -70,6 +69,8 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ rm -rf OpenBLAS ENV OpenBLAS_HOME=/usr/local/openblas-android +WORKDIR /work/build + #################################################################################################### # Specialize base image for ARMv8 @@ -91,3 +92,5 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ cd /usr/local && \ rm -rf OpenBLAS ENV OpenBLAS_HOME=/usr/local/openblas-android + +WORKDIR /work/build diff --git a/ci/docker/Dockerfile.build.arm b/ci/docker/Dockerfile.build.arm index d4b1daad3635..59022c8b4406 100644 --- a/ci/docker/Dockerfile.build.arm +++ b/ci/docker/Dockerfile.build.arm @@ -43,7 +43,6 @@ COPY install/ubuntu_adduser.sh /work/ RUN /work/ubuntu_adduser.sh COPY runtime_functions.sh /work/ -WORKDIR /work/mxnet #################################################################################################### @@ -72,6 +71,8 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ cd /usr/local && \ rm -rf OpenBLAS +WORKDIR /work/mxnet + #################################################################################################### # Specialize base image for ARMv7 @@ -97,6 +98,8 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ cd /usr/local && \ rm -rf OpenBLAS +WORKDIR /work/mxnet + #################################################################################################### # Specialize base image for ARMv8 @@ -121,3 +124,5 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \ make PREFIX=/usr/aarch64-linux-gnu NO_SHARED=1 install && \ cd /usr/local && \ rm -rf OpenBLAS + +WORKDIR /work/mxnet