From 34bb881ba4c9f598ae763a3611718e8e968800d2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 10 Sep 2019 17:29:02 +0200 Subject: [PATCH 1/8] Create a ppc64le image for CUDA builds Borrowing from the ppc64le image, create an image that also supports CUDA builds. --- linux-anvil-ppc64le-cuda/Dockerfile | 43 ++++++++++++++++++++++ linux-anvil-ppc64le-cuda/entrypoint_source | 2 + 2 files changed, 45 insertions(+) create mode 100644 linux-anvil-ppc64le-cuda/Dockerfile create mode 100644 linux-anvil-ppc64le-cuda/entrypoint_source diff --git a/linux-anvil-ppc64le-cuda/Dockerfile b/linux-anvil-ppc64le-cuda/Dockerfile new file mode 100644 index 00000000..bf6dfe1a --- /dev/null +++ b/linux-anvil-ppc64le-cuda/Dockerfile @@ -0,0 +1,43 @@ +FROM ppc64le/centos:7 + +MAINTAINER conda-forge + +# Add qemu in here so that we can use this image on regular linux hosts with qemu user installed +ADD qemu-ppc64le-static /usr/bin/qemu-ppc64le-static + +# Set an encoding to make things work smoothly. +ENV LANG en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 + +# Add a timestamp for the build. Also, bust the cache. +ADD http://worldclockapi.com/api/json/utc/now /opt/docker/etc/timestamp + +# Resolves a nasty NOKEY warning that appears when using yum. +RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le + +# Install basic requirements. +COPY scripts/yum_clean_all /opt/docker/bin/ +RUN yum update -y && \ + yum install -y \ + bzip2 \ + sudo \ + tar \ + which && \ + /opt/docker/bin/yum_clean_all + +# Run common commands +COPY scripts/run_commands /opt/docker/bin/run_commands +RUN /opt/docker/bin/run_commands + +# Add a file for users to source to activate the `conda` +# environment `root`. Also add a file that wraps that for +# use with the `ENTRYPOINT`. +COPY linux-anvil-ppc64le/entrypoint_source /opt/docker/bin/entrypoint_source +COPY scripts/entrypoint /opt/docker/bin/entrypoint + +# Ensure that all containers start with tini and the user selected process. +# Activate the `conda` environment `root`. +# Provide a default command (`bash`), which will start if the user doesn't specify one. +ENTRYPOINT [ "/opt/conda/bin/tini", "--", "/opt/docker/bin/entrypoint" ] +CMD [ "/bin/bash" ] diff --git a/linux-anvil-ppc64le-cuda/entrypoint_source b/linux-anvil-ppc64le-cuda/entrypoint_source new file mode 100644 index 00000000..6dfd4211 --- /dev/null +++ b/linux-anvil-ppc64le-cuda/entrypoint_source @@ -0,0 +1,2 @@ +# Activate the `base` conda environment. +conda activate base From f99ddebcdc31fa3eb55ad44dea207638fef3c155 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 10 Sep 2019 17:37:58 +0200 Subject: [PATCH 2/8] Adjust the ppc64le image to include CUDA support --- linux-anvil-ppc64le-cuda/Dockerfile | 48 +++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/linux-anvil-ppc64le-cuda/Dockerfile b/linux-anvil-ppc64le-cuda/Dockerfile index bf6dfe1a..4f6beda2 100644 --- a/linux-anvil-ppc64le-cuda/Dockerfile +++ b/linux-anvil-ppc64le-cuda/Dockerfile @@ -1,6 +1,12 @@ -FROM ppc64le/centos:7 +ARG CUDA_VER -MAINTAINER conda-forge +FROM nvidia/cuda-ppc64le:${CUDA_VER}-devel-centos7 + +LABEL maintainer="conda-forge " + +# Set CUDA_VER during runtime. +ARG CUDA_VER +ENV CUDA_VER ${CUDA_VER} # Add qemu in here so that we can use this image on regular linux hosts with qemu user installed ADD qemu-ppc64le-static /usr/bin/qemu-ppc64le-static @@ -9,6 +15,9 @@ ADD qemu-ppc64le-static /usr/bin/qemu-ppc64le-static ENV LANG en_US.UTF-8 ENV LANGUAGE=en_US.UTF-8 +# Set path to CUDA install. +ENV CUDA_HOME /usr/local/cuda + # Add a timestamp for the build. Also, bust the cache. ADD http://worldclockapi.com/api/json/utc/now /opt/docker/etc/timestamp @@ -16,6 +25,9 @@ ADD http://worldclockapi.com/api/json/utc/now /opt/docker/etc/timestamp RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-AltArch-7-ppc64le +# Remove preinclude system compilers +RUN rpm -e --nodeps --verbose gcc gcc-c++ + # Install basic requirements. COPY scripts/yum_clean_all /opt/docker/bin/ RUN yum update -y && \ @@ -30,6 +42,38 @@ RUN yum update -y && \ COPY scripts/run_commands /opt/docker/bin/run_commands RUN /opt/docker/bin/run_commands +# Download and cache new compiler packages. +# Should speedup installation of them on CIs. +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate && \ + conda create -n test --yes --quiet --download-only \ + defaults::binutils_impl_linux-ppc64le \ + defaults::binutils_linux-ppc64le \ + defaults::gcc_impl_linux-ppc64le \ + defaults::gcc_linux-ppc64le \ + defaults::gfortran_impl_linux-ppc64le \ + defaults::gfortran_linux-ppc64le \ + defaults::gxx_impl_linux-ppc64le \ + defaults::gxx_linux-ppc64le \ + defaults::libgcc-ng \ + defaults::libgfortran-ng \ + defaults::libstdcxx-ng && \ + conda remove --yes --quiet -n test --all && \ + conda clean -tiy && \ + chgrp -R lucky /opt/conda && \ + chmod -R g=u /opt/conda + +# Download and cache CUDA related packages. +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate && \ + conda create -n test --yes --quiet --download-only \ + defaults::cudatoolkit=${CUDA_VER} \ + defaults::cudnn && \ + conda remove --yes --quiet -n test --all && \ + conda clean -tiy && \ + chgrp -R lucky /opt/conda && \ + chmod -R g=u /opt/conda + # Add a file for users to source to activate the `conda` # environment `root`. Also add a file that wraps that for # use with the `ENTRYPOINT`. From 2aba024541b8e13d12ceeadb230faf2ce3d171c1 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 10 Sep 2019 18:08:41 +0200 Subject: [PATCH 3/8] Build linux-anvil-ppc64le-cuda images on Travis CI --- .travis.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.travis.yml b/.travis.yml index f6d53dd6..58f0eee1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,6 +61,30 @@ matrix: - DOCKERIMAGE=miniforge3 - DOCKERTAG=latest + - os: linux + env: + - DOCKERIMAGE=linux-anvil-ppc64le-cuda + - DOCKERTAG=9.2 + - CUDA_VER=9.2 + + - os: linux + env: + - DOCKERIMAGE=linux-anvil-ppc64le-cuda + - DOCKERTAG=10.0 + - CUDA_VER=10.0 + + - os: linux + env: + - DOCKERIMAGE=linux-anvil-ppc64le-cuda + - DOCKERTAG=10.1 + - CUDA_VER=10.1 + + - os: linux + env: + - DOCKERIMAGE=linux-anvil-ppc64le-cuda + - DOCKERTAG=10.2 + - CUDA_VER=10.2 + env: global: secure: "nM8OaFilQkH14wzD1S6DTGejjo3yL/q/1dpz7144Kw68s8FVqW0zsxCC6960ieokY2LutGSv16qTiIFxnRZnHPCXt7X2MhxcagX8IcMu62DWe2jgqwho0hPI65N/bQYLW1l23e9tjKQxWFZopM4Oyzm6TBqlzibTdbPuQI+YB3RBY0dlkIlupPIYtiNlLRR/HnOyyUny/hg3Z65GWeVpXhiMPqXLlfliTiQ31JgBaNuXiP3/ruSCDeyRPWx62IcPGJ1xVSXL3tvkEI2TpGVCsraKCSbgINhm3AHjQ+8ST6GPMxaOaHrKZzssKJpsZhz1dzWINXTLOQ5LrKtBVwfaevFxDmPEr9RcVlzwAAyuWugCyV4Z6NSt/j2Qqw5qGaiiDHyBH0FMmBgzlPzLZ4JKFsZ68aRkc2qV0MeN0YJRwcQ0EnXRULrcwReBztDHZwixSxqlPpQUwbRr7ne05rBjVoMTKaEhyHPO+KYSwQB1wiQgILBtlP/5ofsYc9Eb46m5JJJhJxuLythKpW9mMqd/US4rQrgEHQ/QRIRYwzGnKf/5WXV3W2o4C9QZpH5Da3J7jOLlqxIY5I+Dv9eEk7XxhT7UaEo7C9tmzjaL2D0yrzPnOnPQhMpmCVNWqdTp1eLcIASKSPbmzz8MuYB5yg48wjXWvDIRBQ6hJyuKHhNGE9k=" From 2ca59df2a4e6138f75334ff4f751e10d2ff0e7bb Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 12 May 2020 20:39:26 -0500 Subject: [PATCH 4/8] Remove cudatoolkit caching for now --- linux-anvil-ppc64le-cuda/Dockerfile | 33 ++++++++++------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/linux-anvil-ppc64le-cuda/Dockerfile b/linux-anvil-ppc64le-cuda/Dockerfile index 4f6beda2..4e1e496e 100644 --- a/linux-anvil-ppc64le-cuda/Dockerfile +++ b/linux-anvil-ppc64le-cuda/Dockerfile @@ -47,28 +47,17 @@ RUN /opt/docker/bin/run_commands RUN source /opt/conda/etc/profile.d/conda.sh && \ conda activate && \ conda create -n test --yes --quiet --download-only \ - defaults::binutils_impl_linux-ppc64le \ - defaults::binutils_linux-ppc64le \ - defaults::gcc_impl_linux-ppc64le \ - defaults::gcc_linux-ppc64le \ - defaults::gfortran_impl_linux-ppc64le \ - defaults::gfortran_linux-ppc64le \ - defaults::gxx_impl_linux-ppc64le \ - defaults::gxx_linux-ppc64le \ - defaults::libgcc-ng \ - defaults::libgfortran-ng \ - defaults::libstdcxx-ng && \ - conda remove --yes --quiet -n test --all && \ - conda clean -tiy && \ - chgrp -R lucky /opt/conda && \ - chmod -R g=u /opt/conda - -# Download and cache CUDA related packages. -RUN source /opt/conda/etc/profile.d/conda.sh && \ - conda activate && \ - conda create -n test --yes --quiet --download-only \ - defaults::cudatoolkit=${CUDA_VER} \ - defaults::cudnn && \ + conda-forge::binutils_impl_linux-ppc64le \ + conda-forge::binutils_linux-ppc64le \ + conda-forge::gcc_impl_linux-ppc64le \ + conda-forge::gcc_linux-ppc64le \ + conda-forge::gfortran_impl_linux-ppc64le \ + conda-forge::gfortran_linux-ppc64le \ + conda-forge::gxx_impl_linux-ppc64le \ + conda-forge::gxx_linux-ppc64le \ + conda-forge::libgcc-ng \ + conda-forge::libgfortran-ng \ + conda-forge::libstdcxx-ng && \ conda remove --yes --quiet -n test --all && \ conda clean -tiy && \ chgrp -R lucky /opt/conda && \ From eb69da14ed3bc8c150184916b8164cc2bc47a924 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 12 May 2020 21:18:57 -0500 Subject: [PATCH 5/8] Remove caching of cudatoolkit as we are not allowed to do so --- linux-anvil-cuda/Dockerfile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/linux-anvil-cuda/Dockerfile b/linux-anvil-cuda/Dockerfile index 40362586..247421bf 100644 --- a/linux-anvil-cuda/Dockerfile +++ b/linux-anvil-cuda/Dockerfile @@ -58,17 +58,6 @@ RUN source /opt/conda/etc/profile.d/conda.sh && \ chgrp -R lucky /opt/conda && \ chmod -R g=u /opt/conda -# Download and cache CUDA related packages. -RUN source /opt/conda/etc/profile.d/conda.sh && \ - conda activate && \ - conda create -n test --yes --quiet --download-only \ - defaults::cudatoolkit=${CUDA_VER} \ - && \ - conda remove --yes --quiet -n test --all && \ - conda clean -tiy && \ - chgrp -R lucky /opt/conda && \ - chmod -R g=u /opt/conda - # Symlink CUDA headers that were moved from $CUDA_HOME/include to /usr/include # in CUDA 10.1. RUN for HEADER_FILE in cublas_api.h cublas.h cublasLt.h cublas_v2.h cublasXt.h nvblas.h; do \ From 930d9d6063e9d31d8593c92171588e00d71ef929 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 27 May 2020 12:46:54 -0500 Subject: [PATCH 6/8] Put cudatoolkit back in for x86_64 @jakirkham and @kkraus14 says that NVIDIA legal team said that this docker image is okay. --- linux-anvil-cuda/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/linux-anvil-cuda/Dockerfile b/linux-anvil-cuda/Dockerfile index 247421bf..f5b17a58 100644 --- a/linux-anvil-cuda/Dockerfile +++ b/linux-anvil-cuda/Dockerfile @@ -57,6 +57,17 @@ RUN source /opt/conda/etc/profile.d/conda.sh && \ conda clean -tiy && \ chgrp -R lucky /opt/conda && \ chmod -R g=u /opt/conda + +# Download and cache CUDA related packages. +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate && \ + conda create -n test --yes --quiet --download-only \ + defaults::cudatoolkit=${CUDA_VER} \ + && \ + conda remove --yes --quiet -n test --all && \ + conda clean -tiy && \ + chgrp -R lucky /opt/conda && \ + chmod -R g=u /opt/conda # Symlink CUDA headers that were moved from $CUDA_HOME/include to /usr/include # in CUDA 10.1. From b8ed14eacef7352e0250d08773b7a3726b6754e9 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 27 May 2020 12:47:18 -0500 Subject: [PATCH 7/8] Remove whitespace --- linux-anvil-cuda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux-anvil-cuda/Dockerfile b/linux-anvil-cuda/Dockerfile index f5b17a58..40362586 100644 --- a/linux-anvil-cuda/Dockerfile +++ b/linux-anvil-cuda/Dockerfile @@ -57,7 +57,7 @@ RUN source /opt/conda/etc/profile.d/conda.sh && \ conda clean -tiy && \ chgrp -R lucky /opt/conda && \ chmod -R g=u /opt/conda - + # Download and cache CUDA related packages. RUN source /opt/conda/etc/profile.d/conda.sh && \ conda activate && \ From 5a7a81ab56c9b7064527821b4468fab5a8609bd1 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Wed, 27 May 2020 12:48:53 -0500 Subject: [PATCH 8/8] Add a comment on the ppc64le docker image --- linux-anvil-ppc64le-cuda/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/linux-anvil-ppc64le-cuda/Dockerfile b/linux-anvil-ppc64le-cuda/Dockerfile index 4e1e496e..2d64119a 100644 --- a/linux-anvil-ppc64le-cuda/Dockerfile +++ b/linux-anvil-ppc64le-cuda/Dockerfile @@ -1,3 +1,7 @@ +# Note that this image doesn't cache cudatoolkit as it is not packaged by defaults. +# This docker image is meant for packages using CUDA driver and not for packages +# using the CUDA runtime. + ARG CUDA_VER FROM nvidia/cuda-ppc64le:${CUDA_VER}-devel-centos7