diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 6c880988eceeb..89ebb64dabbd9 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -3400,20 +3400,7 @@ function run_setup(){ SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then echo "Using python abi: $1" - if [ "$1" == "cp37-cp37m" ]; then - if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then - export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/ - export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.7/lib/ - export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH} - #after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export - export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 - export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/ - export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib - pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt - else - exit 1 - fi - elif [ "$1" == "cp38-cp38" ]; then + if [ "$1" == "cp38-cp38" ]; then if [ -d "/Library/Frameworks/Python.framework/Versions/3.8" ]; then export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/ export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.8/lib/ @@ -3465,19 +3452,24 @@ function run_setup(){ else exit 1 fi + elif [ "$1" == "cp312-cp312" ]; then + if [ -d "/Library/Frameworks/Python.framework/Versions/3.12" ]; then + export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.12/lib/ + export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.12/lib/ + export PATH=/Library/Frameworks/Python.framework/Versions/3.12/bin/:${PATH} + #after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export + export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.12/bin/python3 + export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.12/include/python3.12/ + export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.12/lib/libpython3.12.dylib + pip3.12 install --user -r ${PADDLE_ROOT}/python/requirements.txt + else + exit 1 + fi fi else if [ "$1" != "" ]; then echo "using python abi: $1" - if [ "$1" == "cp37-cp37m" ]; then - export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH} - export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH} - #after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export - export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.7.0/bin/python3.7 - export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.7.0/include/python3.7m - export PYTHON_LIBRARIES=/opt/_internal/cpython-3.7.0/lib/libpython3.so - pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt - elif [ "$1" == "cp38-cp38" ]; then + if [ "$1" == "cp38-cp38" ]; then export LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} export PATH=/opt/_internal/cpython-3.8.0/bin/:${PATH} #after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export @@ -3509,14 +3501,14 @@ function run_setup(){ export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.11.0/include/python3.11 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.11.0/lib/libpython3.so pip3.11 install -r ${PADDLE_ROOT}/python/requirements.txt - elif [ "$1" == "conda-python3.7" ]; then - export LD_LIBRARY_PATH=/opt/conda/lib/:${LD_LIBRARY_PATH} - export PATH=/opt/conda/bin/:${PATH} + elif [ "$1" == "cp312-cp312" ]; then + export LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} + export PATH=/opt/_internal/cpython-3.12.0/bin/:${PATH} #after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export - export DPYTHON_EXECUTABLE=/opt/conda/bin/python - export PYTHON_INCLUDE_DIR=/opt/conda/include/python3.7m - export PYTHON_LIBRARIES=/opt/conda/lib/libpython3.so - /opt/conda/bin/pip install -r ${PADDLE_ROOT}/python/requirements.txt + export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.12.0/bin/python3.12 + export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.12.0/include/python3.12 + export PYTHON_LIBRARIES=/opt/_internal/cpython-3.12.0/lib/libpython3.so + pip3.12 install -r ${PADDLE_ROOT}/python/requirements.txt fi else pip install -r ${PADDLE_ROOT}/python/requirements.txt diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt index b7866aa46109d..15cf679177709 100644 --- a/python/unittest_py/requirements.txt +++ b/python/unittest_py/requirements.txt @@ -13,9 +13,9 @@ prettytable distro numpy>=1.20 autograd==1.4 -librosa==0.8.1 +librosa==0.8.1 ; python_version<"3.12" parameterized -wandb>=0.13 +wandb>=0.13 ; python_version<"3.12" xlsxwriter==3.0.9 xdoctest==1.1.1 ubelt==1.3.3 # just for xdoctest diff --git a/setup.py b/setup.py index 221e0a0770e06..4f61bb9727b7a 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ import sys import time from contextlib import contextmanager -from distutils.spawn import find_executable from subprocess import CalledProcessError from setuptools import Command, Extension, setup @@ -58,7 +57,7 @@ ) # check cmake -CMAKE = find_executable('cmake3') or find_executable('cmake') +CMAKE = shutil.which('cmake3') or shutil.which('cmake') assert ( CMAKE ), 'The "cmake" executable is not found. Please check if Cmake is installed.' diff --git a/tools/dockerfile/Dockerfile.centos b/tools/dockerfile/Dockerfile.centos index 63f8128014d35..c9ae968c920c5 100644 --- a/tools/dockerfile/Dockerfile.centos +++ b/tools/dockerfile/Dockerfile.centos @@ -13,10 +13,10 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig -RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel +RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel patch COPY build_scripts /build_scripts RUN bash build_scripts/build.sh -RUN bash build_scripts/install_nccl2.sh +#RUN bash build_scripts/install_nccl2.sh RUN bash build_scripts/install_trt.sh RUN rm -rf build_scripts RUN ln -s /usr/local/ssl/include/openssl /usr/include @@ -53,19 +53,22 @@ RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/re RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install setuptools -U && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install setuptools -U && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install setuptools -U && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools -U + LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools -U && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install setuptools -U RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install -r /root/requirements.txt && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install -r /root/requirements.txt && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install -r /root/requirements.txt && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install -r /root/requirements.txt && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install -r /root/requirements.txt && \ go get github.com/Masterminds/glide && \ rm -rf /root/requirements.txt RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ - LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' + LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \ + LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install pre-commit 'ipython==5.3.0' RUN wget -O /opt/swig-2.0.12.tar.gz --no-check-certificate https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz diff --git a/tools/dockerfile/Dockerfile.ubuntu20 b/tools/dockerfile/Dockerfile.ubuntu20 index 3fbbaf8f707af..bfeb761f62fcd 100644 --- a/tools/dockerfile/Dockerfile.ubuntu20 +++ b/tools/dockerfile/Dockerfile.ubuntu20 @@ -24,7 +24,8 @@ RUN rm /etc/apt/sources.list.d/* RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub RUN apt-get update --allow-unauthenticated && \ - apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip \ coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx \ @@ -47,7 +48,7 @@ RUN ln -s /usr/local/gcc-12.1/bin/gcc /usr/bin/gcc RUN ln -s /usr/local/gcc-12.1/bin/g++ /usr/bin/g++ ENV PATH=/usr/local/gcc-12.1/bin:$PATH -Run bash /build_scripts/install_cudnn.sh cudnn841 +RUN bash /build_scripts/install_cudnn.sh cudnn841 ENV CUDNN_VERSION=8.4.1 #RUN bash /build_scripts/install_nccl2.sh RUN rm -rf /build_script @@ -58,37 +59,40 @@ RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && ta ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH RUN apt-get update && \ - apt-get install -y python3.7 python3.7-dev python3.7-distutils\ - python3.8 python3.8-dev python3.8-distutils \ + apt-get install -y python3.8 python3.8-dev python3.8-distutils \ python3.9 python3.9-dev python3.9-distutils \ python3.10 python3.10-dev python3.10-distutils \ - python3.11 python3.11-dev python3.11-distutils && \ + python3.11 python3.11-dev python3.11-distutils \ + python3.12 python3.12-dev python3.12-distutils && \ apt-get install python-is-python3 && \ rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \ rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3 WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip +RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip WORKDIR /home/setuptools-50.3.2 RUN python3.9 setup.py build && python3.9 setup.py install && \ - python3.8 setup.py build && python3.8 setup.py install && \ - python3.7 setup.py build && python3.7 setup.py install && \ - python3.10 setup.py build && python3.10 setup.py install && \ - python3.11 setup.py build && python3.11 setup.py install + python3.8 setup.py build && python3.8 setup.py install WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz -WORKDIR pip-20.0.1 +RUN wget https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz +WORKDIR /home/setuptools-68.2.2 +RUN python3.10 setup.py build && python3.10 setup.py install && \ + python3.11 setup.py build && python3.11 setup.py install && \ + python3.12 setup.py build && python3.12 setup.py install +WORKDIR /home +RUN wget https://files.pythonhosted.org/packages/1f/7f/4da15e07ccd11c84c1ccc8f6e24288d5e76c99441bf80e315b33542db951/pip-23.3.1.tar.gz && tar -zxf pip-23.3.1.tar.gz +WORKDIR pip-23.3.1 RUN python3.9 setup.py install && \ - python3.8 setup.py install && \ - python3.7 setup.py install && \ + python3.8 setup.py install && \ python3.10 setup.py install && \ - python3.11 setup.py install + python3.11 setup.py install && \ + python3.12 setup.py install WORKDIR /home -RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \ - rm -r setuptools-50.3.2 pip-20.0.1 +RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz pip-23.3.1.tar.gz && \ + rm -r setuptools-50.3.2 setuptools-68.2.2 pip-23.3.1 # remove them when apt-get support 2.27 and higher version RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \ @@ -115,56 +119,49 @@ RUN git config --global credential.helper store RUN localedef -i en_US -f UTF-8 en_US.UTF-8 #For pre-commit -RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \ - python3.8 -m pip --no-cache-dir install --upgrade pip && \ - python3.9 -m pip --no-cache-dir install --upgrade pip && \ - python3.10 -m pip --no-cache-dir install --upgrade pip && \ - python3.11 -m pip --no-cache-dir install --upgrade pip - RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \ rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3 -RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \ - pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.8 --no-cache-dir install ipython==5.3.0 && \ +RUN pip3.8 --no-cache-dir install ipython==5.3.0 && \ pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.9 --no-cache-dir install ipython==5.3.0 && \ pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.10 --no-cache-dir install ipython==5.3.0 && \ pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel && \ pip3.11 --no-cache-dir install ipython==5.3.0 && \ - pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel + pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel && \ + pip3.12 --no-cache-dir install ipython==5.3.0 && \ + pip3.12 --no-cache-dir install ipykernel==4.6.0 wheel #For docstring checker -RUN pip3.7 --no-cache-dir install pytest astroid isort && \ - pip3.8 --no-cache-dir install pytest astroid isort && \ +RUN pip3.8 --no-cache-dir install pytest astroid isort && \ pip3.9 --no-cache-dir install pytest astroid isort && \ pip3.10 --no-cache-dir install pytest astroid isort && \ - pip3.11 --no-cache-dir install pytest astroid isort + pip3.11 --no-cache-dir install pytest astroid isort && \ + pip3.12 --no-cache-dir install pytest astroid isort -RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ - pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ +RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \ - pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ - pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 + pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ + pip3.12 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 COPY ./python/requirements.txt /root/ COPY ./python/unittest_py/requirements.txt /home/ -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \ - pip3.7 --no-cache-dir install -r /home/requirements.txt && \ - pip3.8 --no-cache-dir install -r /root/requirements.txt && \ +RUN pip3.8 --no-cache-dir install -r /root/requirements.txt && \ pip3.8 --no-cache-dir install -r /home/requirements.txt && \ pip3.9 --no-cache-dir install -r /root/requirements.txt && \ pip3.9 --no-cache-dir install -r /home/requirements.txt && \ pip3.10 --no-cache-dir install -r /root/requirements.txt && \ pip3.10 --no-cache-dir install -r /home/requirements.txt && \ pip3.11 --no-cache-dir install -r /root/requirements.txt && \ - pip3.11 --no-cache-dir install -r /home/requirements.txt + pip3.11 --no-cache-dir install -r /home/requirements.txt && \ + pip3.12 --no-cache-dir install -r /root/requirements.txt && \ + pip3.12 --no-cache-dir install -r /home/requirements.txt # ccache 4.2.0 diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh index 680f9c710cc39..cb17a76a1dd05 100644 --- a/tools/dockerfile/build_scripts/build.sh +++ b/tools/dockerfile/build_scripts/build.sh @@ -24,7 +24,7 @@ set -ex # remove others to expedite build and reduce docker image size. The original # manylinux docker image project builds many python versions. # NOTE We added back 3.5.1, since auditwheel requires python 3.3+ -CPYTHON_VERSIONS="3.11.0 3.10.0 3.9.0 3.8.0" +CPYTHON_VERSIONS="3.12.0 3.11.0 3.10.0 3.9.0 3.8.0" # openssl version to build, with expected sha256 hash of .tar.gz # archive @@ -82,6 +82,7 @@ PY38_BIN=/opt/python/cp38-cp38/bin PY39_BIN=/opt/python/cp39-cp39/bin PY310_BIN=/opt/python/cp310-cp310/bin PY311_BIN=/opt/python/cp311-cp311/bin +PY312_BIN=/opt/python/cp312-cp312/bin # NOTE Since our custom manylinux image builds pythons with shared # libpython, we need to add libpython's dir to LD_LIBRARY_PATH before running # python. diff --git a/tools/dockerfile/build_scripts/build_utils.sh b/tools/dockerfile/build_scripts/build_utils.sh index 7f86f2ca863b7..10088cd2c5b02 100755 --- a/tools/dockerfile/build_scripts/build_utils.sh +++ b/tools/dockerfile/build_scripts/build_utils.sh @@ -83,9 +83,6 @@ function do_cpython_build { rm -rf Python-$py_ver # Some python's install as bin/python3. Make them available as # bin/python. - if [ -e ${prefix}/bin/python3.7 ]; then - ln -s python3.7 ${prefix}/bin/python - fi if [ -e ${prefix}/bin/python3.8 ]; then ln -s python3.8 ${prefix}/bin/python fi @@ -98,16 +95,20 @@ function do_cpython_build { if [ -e ${prefix}/bin/python3.11 ]; then ln -s python3.11 ${prefix}/bin/python fi + if [ -e ${prefix}/bin/python3.12 ]; then + ln -s python3.12 ${prefix}/bin/python + fi # NOTE Make libpython shared library visible to python calls below - if [ -e ${prefix}/bin/python3.10 ] || [ -e ${prefix}/bin/python3.11 ]; then + if [ -e ${prefix}/bin/python3.10 ] || [ -e ${prefix}/bin/python3.11 ] || [ -e ${prefix}/bin/python3.12 ]; then LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.trusted-host mirrors.aliyun.com LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.index-url http://mirrors.aliyun.com/pypi/simple/ fi LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py - LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2 + LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.40.0 cd / ls ${MY_DIR} - local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + abi_version=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python -V|awk '{print $2}'|awk -F '.' '{print $1$2}') + local abi_tag=$(echo cp$abi_version-cp$abi_version) ln -s ${prefix} /opt/python/${abi_tag} } diff --git a/tools/dockerfile/build_scripts/install_cudnn.sh b/tools/dockerfile/build_scripts/install_cudnn.sh index 0ee6d1e160e81..e0922a1435413 100644 --- a/tools/dockerfile/build_scripts/install_cudnn.sh +++ b/tools/dockerfile/build_scripts/install_cudnn.sh @@ -45,6 +45,14 @@ elif [[ "$1" == "cudnn821" && "$VERSION" == "11.2" ]]; then cp -r lib64 /usr && cd ../ && \ rm -f cudnn-11.3-linux-x64-v8.2.1.32.tgz && \ rm -rf cuda +elif [[ "$1" == "cudnn841" && "$VERSION" == "11.6" ]]; then + wget -q https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz --no-check-certificate + tar xJvf cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz && \ + cd cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive && \ + cp -r include /usr && \ + cp -r lib /usr && cd ../ && \ + rm -f cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz && \ + rm -rf cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive elif [[ "$1" == "cudnn841" && "$VERSION" == "11.7" ]]; then wget -q https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz --no-check-certificate tar xJvf cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz && \ @@ -53,12 +61,20 @@ elif [[ "$1" == "cudnn841" && "$VERSION" == "11.7" ]]; then cp -r lib /usr && cd ../ && \ rm -f cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz && \ rm -rf cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive +elif [[ "$1" == "cudnn860" && "$VERSION" == "11.8" ]]; then + wget -q https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-8.6.0.163_cuda11-archive.tar.xz --no-check-certificate + tar xJvf cudnn-linux-x86_64-8.6.0.163_cuda11-archive.tar.xz + cd cudnn-linux-x86_64-8.6.0.163_cuda11-archive + cp -r include /usr + cp -r lib /usr && cd ../ + rm -f cudnn-linux-x86_64-8.6.0.163_cuda11-archive.tar.xz + rm -rf cudnn-linux-x86_64-8.6.0.163_cuda11-archive elif [[ "$1" == "cudnn891" && "$VERSION" == "12.0" ]]; then wget https://paddle-ci.gz.bcebos.com/cudnn/cudnn-linux-x86_64-8.9.1.23_cuda12-archive.tar.xz --no-check-certificate - tar xJvf cudnn-linux-x86_64-8.9.1.23_cuda12-archive.tar.xz && \ - cd cudnn-linux-x86_64-8.9.1.23_cuda12-archive && \ - cp -r include /usr && \ - cp -r lib /usr && cd ../ && \ - rm -f cudnn-linux-x86_64-8.9.1.23_cuda12-archive.tar.xz && \ + tar xJvf cudnn-linux-x86_64-8.9.1.23_cuda12-archive.tar.xz + cd cudnn-linux-x86_64-8.9.1.23_cuda12-archive + cp -r include /usr + cp -r lib /usr && cd ../ + rm -f cudnn-linux-x86_64-8.9.1.23_cuda12-archive.tar.xz rm -rf cudnn-linux-x86_64-8.9.1.23_cuda12-archive fi diff --git a/tools/dockerfile/build_scripts/install_gcc.sh b/tools/dockerfile/build_scripts/install_gcc.sh index abee06a42dded..4451e2783bb6b 100644 --- a/tools/dockerfile/build_scripts/install_gcc.sh +++ b/tools/dockerfile/build_scripts/install_gcc.sh @@ -31,7 +31,7 @@ fi if [ "$1" == "gcc82" ]; then wget -q --no-proxy https://paddle-ci.gz.bcebos.com/gcc-8.2.0.tar.xz - tar -xvf gcc-8.2.0.tar.xz && \ + tar -xf gcc-8.2.0.tar.xz && \ cd gcc-8.2.0 && \ wget -q --no-proxy https://paddle-ci.gz.bcebos.com/sanitizer_platform_limits_posix.cc.patch wget -q --no-proxy https://paddle-ci.gz.bcebos.com/sanitizer_platform_limits_posix.h.patch @@ -43,6 +43,12 @@ if [ "$1" == "gcc82" ]; then ../gcc-8.2.0/configure --prefix=/usr/local/gcc-8.2 --enable-threads=posix --disable-checking --disable-multilib && \ make -j8 && make install cd .. && rm -rf temp_gcc82 gcc-8.2.0 gcc-8.2.0.tar.xz + if [ -f "/etc/redhat-release" ];then + cp ${lib_so_6} ${lib_so_6}.bak && rm -f ${lib_so_6} && + ln -s /usr/local/gcc-8.2/lib64/libgfortran.so.5 ${lib_so_5} && \ + ln -s /usr/local/gcc-8.2/lib64/libstdc++.so.6 ${lib_so_6} && \ + cp /usr/local/gcc-8.2/lib64/libstdc++.so.6.0.25 ${lib_path} + fi elif [ "$1" == "gcc122" ]; then wget -q --no-proxy https://paddle-ci.gz.bcebos.com/gcc-12.2.0.tar.gz tar -xzf gcc-12.2.0.tar.gz && \ diff --git a/tools/dockerfile/build_scripts/install_nccl2.sh b/tools/dockerfile/build_scripts/install_nccl2.sh index 97c4875a2a4b9..2680910834023 100644 --- a/tools/dockerfile/build_scripts/install_nccl2.sh +++ b/tools/dockerfile/build_scripts/install_nccl2.sh @@ -23,9 +23,9 @@ elif [ "$VERSION" == "10.2" ] || [ "$VERSION" == "10.1" ] || [ "$VERSION" == "11 wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-2.10.3-1+cuda11.4.x86_64.rpm wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-devel-2.10.3-1+cuda11.4.x86_64.rpm wget --no-check-certificate -q https://nccl2-deb.cdn.bcebos.com/libnccl-static-2.10.3-1+cuda11.4.x86_64.rpm - rpm -Fivh libnccl-2.10.3-1+cuda11.4.x86_64.rpm - rpm -Fivh libnccl-devel-2.10.3-1+cuda11.4.x86_64.rpm - rpm -Fivh libnccl-static-2.10.3-1+cuda11.4.x86_64.rpm && rm -f libnccl-* + rpm -ivh libnccl-2.10.3-1+cuda11.4.x86_64.rpm + rpm -ivh libnccl-devel-2.10.3-1+cuda11.4.x86_64.rpm + rpm -ivh libnccl-static-2.10.3-1+cuda11.4.x86_64.rpm && rm -f libnccl-* exit 0 fi DEB="nccl-repo-ubuntu1804-2.10.3-cuda11.4_1.0-1_amd64.deb" diff --git a/tools/dockerfile/build_scripts/install_trt.sh b/tools/dockerfile/build_scripts/install_trt.sh index 3cf305ea63d44..0fdadc8be8d70 100644 --- a/tools/dockerfile/build_scripts/install_trt.sh +++ b/tools/dockerfile/build_scripts/install_trt.sh @@ -31,21 +31,31 @@ if [[ "$1" == "trt8034" && "$VERSION" == "11.2" ]];then tar -zxf TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-11.3.cudnn8.2.tar.gz -C /usr/local cp -rf /usr/local/TensorRT-8.0.3.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.0.3.4/lib/* /usr/lib/ rm TensorRT-8.0.3.4.Linux.x86_64-gnu.cuda-11.3.cudnn8.2.tar.gz +elif [[ "$1" == "trt8424" ]];then + wget https://paddle-qa.bj.bcebos.com/nvidia/trt/TensorRT-8.4.2.4.tgz --no-check-certificate + tar -zxf TensorRT-8.4.2.4.tgz -C /usr/local + cp -rf /usr/local/TensorRT-8.4.2.4/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.4.2.4/lib/* /usr/lib/ + rm -f TensorRT-8.4.2.4.tgz +elif [[ "$1" == "trt8616" ]];then + wget -q --no-proxy https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz --no-check-certificate + tar -zxf TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-8.6.1.6/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.6.1.6/lib/* /usr/lib/ + rm -f TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz elif [[ "$1" == "trt8406" ]];then - wget -q --no-proxy https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz --no-check-certificate - tar -zxf TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz -C /usr/local - cp -rf /usr/local/TensorRT-8.4.0.6/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.4.0.6/lib/* /usr/lib/ - rm -f TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz + wget -q --no-proxy https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz --no-check-certificate + tar -zxf TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-8.4.0.6/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.4.0.6/lib/* /usr/lib/ + rm -f TensorRT-8.4.0.6.Linux.x86_64-gnu.cuda-11.6.cudnn8.3.tar.gz elif [[ "$1" == "trt8431" ]];then - wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz --no-check-certificate --no-proxy - tar -zxf TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz -C /usr/local - cp -rf /usr/local/TensorRT-8.4.3.1/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.4.3.1/lib/* /usr/lib/ - rm -f TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz --no-check-certificate --no-proxy + tar -zxf TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-8.4.3.1/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.4.3.1/lib/* /usr/lib/ + rm -f TensorRT-8.4.3.1.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz elif [[ "$1" == "trt8531" ]];then - wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz --no-check-certificate --no-proxy - tar -zxf TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz -C /usr/local - cp -rf /usr/local/TensorRT-8.5.3.1/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.5.3.1/lib/* /usr/lib/ - rm -f TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz + wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz --no-check-certificate --no-proxy + tar -zxf TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz -C /usr/local + cp -rf /usr/local/TensorRT-8.5.3.1/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.5.3.1/lib/* /usr/lib/ + rm -f TensorRT-8.5.3.1.Linux.x86_64-gnu.cuda-11.8.cudnn8.6.tar.gz elif [[ "$VERSION" == "11.2" ]];then wget -q --no-proxy https://paddle-ci.gz.bcebos.com/TRT/TensorRT7-cuda11.1-cudnn8.1.tar.gz --no-check-certificate tar -zxf TensorRT7-cuda11.1-cudnn8.1.tar.gz -C /usr/local diff --git a/tools/dockerfile/build_scripts/python-tag-abi-tag.py b/tools/dockerfile/build_scripts/python-tag-abi-tag.py deleted file mode 100644 index 79f906c3e1076..0000000000000 --- a/tools/dockerfile/build_scripts/python-tag-abi-tag.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Utility script to print the python tag + the abi tag for a Python -# See PEP 425 for exactly what these are, but an example would be: -# cp27-cp27mu - -from wheel.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver - -print(f"{get_abbr_impl()}{get_impl_ver()}-{get_abi_tag()}") diff --git a/tools/dockerfile/centos7_manylinux.sh b/tools/dockerfile/centos7_manylinux.sh index 4beb8b3a592ad..2474cbf2c2779 100755 --- a/tools/dockerfile/centos7_manylinux.sh +++ b/tools/dockerfile/centos7_manylinux.sh @@ -18,184 +18,58 @@ set -xe REPO="${REPO:-paddledocker}" -function make_cuda9cudnn7(){ - sed 's//9.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - -} - - -function make_cuda10cudnn7() { - sed 's//10.0-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - -} - - -function make_cuda101cudnn7() { - sed 's//10.1-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda102cudnn7() { - sed 's//10.2-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda102cudnn7gcc54() { - sed 's//10.2-cudnn7-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda102cudnn8() { - sed 's//10.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda102cudnn81gcc54 { - sed 's//10.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRun yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn811 \nENV CUDNN_VERSION=8.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda102cudnn81gcc82 { - sed 's//10.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRun yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn811 \nENV CUDNN_VERSION=8.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda11cudnn8() { - sed 's//11.0-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda111cudnn81gcc54() { - sed 's//11.1-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/install_nccl2.sh#\n#g" Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRun yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn811 \nENV CUDNN_VERSION=8.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda111cudnn81gcc82() { - sed 's//11.1-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/install_nccl2.sh#\n#g" Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRun yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn811 \nENV CUDNN_VERSION=8.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda112cudnn8() { - sed 's//11.2.1-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda112cudnn821trt8034gcc82() { +function make_cuda112cudnn821trt8034() { sed 's//11.2.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn821 \nENV CUDNN_VERSION=8.2.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - sed -i "/install_trt.sh/d" Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_trt.sh trt8034 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRUN bash build_scripts/install_cudnn.sh cudnn821 \nENV CUDNN_VERSION=8.2.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8034#g" Dockerfile.tmp sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp } -function make_cuda112cudnn821trt8034gcc54() { - sed 's//11.2.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc54 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-5.4/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-5.4/bin:\$PATH \nRUN yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRun bash build_scripts/install_cudnn.sh cudnn821 \nENV CUDNN_VERSION=8.2.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - sed -i "/install_trt.sh/d" Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_trt.sh trt8034 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp -} - -function make_cuda113cudnn8() { - sed 's//11.3.1-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda114cudnn8() { - sed 's//11.4.3-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp -} - -function make_cuda115cudnn8() { - sed 's//11.5.1-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp +function make_cuda116cudnn840trt8406() { + sed 's//11.6.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8406#g" Dockerfile.tmp + sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp } -function make_cuda116cudnn8() { - sed 's//11.6.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp +function make_cuda117cudnn841trt8424() { + sed 's//11.7.1-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn841 \nENV CUDNN_VERSION=8.4.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8424#g" Dockerfile.tmp + sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp } -function make_cuda116cudnn840trt8406gcc82() { - sed 's//11.6.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp - sed -i "/install_trt.sh/d" Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_trt.sh trt8406 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp +function make_cuda118cudnn860trt8531() { + sed 's//11.8.0-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn860 \nENV CUDNN_VERSION=8.6.0 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8531#g" Dockerfile.tmp sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp } -function make_cuda117cudnn841() { - sed 's//11.7.0-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp - sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRun bash build_scripts/install_cudnn.sh cudnn841 \nENV CUDNN_VERSION=8.4.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp +function make_cuda120cudnn891trt8616() { + sed 's//12.0.1-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp + sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn891 \nENV CUDNN_VERSION=8.9.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp + sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp } function main() { local CMD=$1 case $CMD in - cuda9cudnn7) - make_cuda9cudnn7 - ;; - cuda10cudnn7) - make_cuda10cudnn7 - ;; - cuda101cudnn7) - make_cuda101cudnn7 - ;; - cuda102cudnn7) - make_cuda102cudnn7 - ;; - cuda102cudnn7gcc54) - make_cuda102cudnn7gcc54 - ;; - cuda102cudnn8) - make_cuda102cudnn8 - ;; - cuda102cudnn811gcc54) - make_cuda102cudnn81gcc54 - ;; - cuda102cudnn811gcc82) - make_cuda102cudnn81gcc82 - ;; - cuda11cudnn8) - make_cuda11cudnn8 - ;; - cuda111cudnn81gcc54) - make_cuda111cudnn81gcc54 - ;; - cuda111cudnn81gcc82) - make_cuda111cudnn81gcc82 - ;; - cuda112cudnn8) - make_cuda112cudnn8 - ;; - cuda112cudnn821trt8034gcc82) - make_cuda112cudnn821trt8034gcc82 - ;; - cuda112cudnn821trt8034gcc54) - make_cuda112cudnn821trt8034gcc54 + cuda112cudnn821trt8034) + make_cuda112cudnn821trt8034 ;; - cuda113cudnn8) - make_cuda113cudnn8 - ;; - cuda114cudnn8) - make_cuda114cudnn8 - ;; - cuda115cudnn8) - make_cuda115cudnn8 + cuda116cudnn840trt8406) + make_cuda116cudnn840trt8406 ;; - cuda116cudnn8) - make_cuda116cudnn8 + cuda117cudnn841trt8424) + make_cuda117cudnn841trt8424 ;; - cuda116cudnn840trt8406gcc82) - make_cuda116cudnn840trt8406gcc82 + cuda118cudnn860trt8531) + make_cuda118cudnn860trt8531 ;; - cuda117cudnn841) - make_cuda117cudnn841 + cuda120cudnn891trt8616) + make_cuda120cudnn891trt8616 ;; *) echo "Make dockerfile error, Without this paramet." diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh index b9487be10022e..c3a588c537992 100644 --- a/tools/dockerfile/ci_dockerfile.sh +++ b/tools/dockerfile/ci_dockerfile.sh @@ -36,7 +36,7 @@ function make_ubuntu_trt7_dockerfile(){ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro \&\& pip3.8 install PyGithub distro" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro" ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ COPY tools/dockerfile/build_scripts /build_scripts \\ RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\ @@ -116,7 +116,7 @@ function make_unbuntu20_cu12_dockerfile(){ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel distro \&\& pip3.7 install PyGithub \&\& pip3.8 install distro" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel distro \&\& pip3.8 install distro" ${dockerfile_name} sed -i 's# && rm /etc/apt/sources.list.d/nvidia-ml.list##g' ${dockerfile_name} sed -i 's#Run bash /build_scripts/install_cudnn.sh cudnn841#RUN bash /build_scripts/install_cudnn.sh cudnn891#g' ${dockerfile_name} } @@ -135,7 +135,7 @@ function make_ubuntu20_cu112_dockerfile(){ cd git-2.17.1 \&\& \ ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.7 install PyGithub distro \&\& pip3.8 install PyGithub distro" ${dockerfile_name} + sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro" ${dockerfile_name} sed -i 's###g' ${dockerfile_name} sed -i "s##WORKDIR /usr/bin \\ ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} diff --git a/tools/dockerfile/ubuntu20_dev.sh b/tools/dockerfile/ubuntu20_dev.sh new file mode 100755 index 0000000000000..85c46a1416e22 --- /dev/null +++ b/tools/dockerfile/ubuntu20_dev.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function base_image(){ + if [[ ${ref_CUDA_MAJOR} == "11.2" ]];then + dockerfile_name="Dockerfile-112" + sed "s##nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-11.2/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} + sed -i "s##WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} + sed -i "s#gcc121#gcc82#g" ${dockerfile_name} + sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name} + sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8034#g' ${dockerfile_name} + sed -i 's#cudnn841#cudnn821#g' ${dockerfile_name} + sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.2.1#g' ${dockerfile_name} + elif [[ ${ref_CUDA_MAJOR} == "11.6" ]];then + dockerfile_name="Dockerfile-116" + sed "s##nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-11.7/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} + sed -i "s##WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} + sed -i "s#gcc121#gcc82#g" ${dockerfile_name} + sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name} + sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8406#g' ${dockerfile_name} + elif [[ ${ref_CUDA_MAJOR} == "11.7" ]];then + dockerfile_name="Dockerfile-117" + sed "s##nvidia/cuda:11.7.1-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-11.7/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} + sed -i "s##WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} + sed -i "s#gcc121#gcc82#g" ${dockerfile_name} + sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name} + sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8424#g' ${dockerfile_name} + elif [[ ${ref_CUDA_MAJOR} == "11.8" ]];then + dockerfile_name="Dockerfile-118" + sed "s##nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} + sed -i "s##WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} + sed -i "s#gcc121#gcc82#g" ${dockerfile_name} + sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name} + sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name} + sed -i 's#cudnn841#cudnn860#g' ${dockerfile_name} + sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.6.0#g' ${dockerfile_name} + elif [[ ${ref_CUDA_MAJOR} == "12.0" ]];then + dockerfile_name="Dockerfile-120" + sed "s##nvidia/cuda:12.0.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} + sed -i "s##ENV LD_LIBRARY_PATH=/usr/local/cuda-12.0/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name} + sed -i 's###g' ${dockerfile_name} + sed -i "s##WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name} + sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name} + sed -i 's#cudnn841#cudnn891#g' ${dockerfile_name} + sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.9.1#g' ${dockerfile_name} + else + echo "Dockerfile ERROR!!!" + exit 1 + fi + +} + + +export ref_CUDA_MAJOR=11.2 +base_image +export ref_CUDA_MAJOR=11.6 +base_image +export ref_CUDA_MAJOR=11.7 +base_image +export ref_CUDA_MAJOR=11.8 +base_image +export ref_CUDA_MAJOR=12.0 +base_image