Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add python3.12 In Docker #58069

Merged
merged 20 commits into from
Nov 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 22 additions & 30 deletions paddle/scripts/paddle_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3400,20 +3400,7 @@ function run_setup(){
SYSTEM=`uname -s`
if [ "$SYSTEM" == "Darwin" ]; then
echo "Using python abi: $1"
if [ "$1" == "cp37-cp37m" ]; then
if [ -d "/Library/Frameworks/Python.framework/Versions/3.7" ]; then
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/
export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.7/lib/
export PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin/:${PATH}
#after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export
export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3
export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/
export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib
pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt
else
exit 1
fi
elif [ "$1" == "cp38-cp38" ]; then
if [ "$1" == "cp38-cp38" ]; then
if [ -d "/Library/Frameworks/Python.framework/Versions/3.8" ]; then
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/
export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.8/lib/
Expand Down Expand Up @@ -3465,19 +3452,24 @@ function run_setup(){
else
exit 1
fi
elif [ "$1" == "cp312-cp312" ]; then
if [ -d "/Library/Frameworks/Python.framework/Versions/3.12" ]; then
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.12/lib/
export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:/Library/Frameworks/Python.framework/Versions/3.12/lib/
export PATH=/Library/Frameworks/Python.framework/Versions/3.12/bin/:${PATH}
#after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export
export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.12/bin/python3
export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.12/include/python3.12/
export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.12/lib/libpython3.12.dylib
pip3.12 install --user -r ${PADDLE_ROOT}/python/requirements.txt
else
exit 1
fi
fi
else
if [ "$1" != "" ]; then
echo "using python abi: $1"
if [ "$1" == "cp37-cp37m" ]; then
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH}
export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH}
#after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export
export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.7.0/bin/python3.7
export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.7.0/include/python3.7m
export PYTHON_LIBRARIES=/opt/_internal/cpython-3.7.0/lib/libpython3.so
pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
elif [ "$1" == "cp38-cp38" ]; then
if [ "$1" == "cp38-cp38" ]; then
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH}
export PATH=/opt/_internal/cpython-3.8.0/bin/:${PATH}
#after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export
Expand Down Expand Up @@ -3509,14 +3501,14 @@ function run_setup(){
export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.11.0/include/python3.11
export PYTHON_LIBRARIES=/opt/_internal/cpython-3.11.0/lib/libpython3.so
pip3.11 install -r ${PADDLE_ROOT}/python/requirements.txt
elif [ "$1" == "conda-python3.7" ]; then
export LD_LIBRARY_PATH=/opt/conda/lib/:${LD_LIBRARY_PATH}
export PATH=/opt/conda/bin/:${PATH}
elif [ "$1" == "cp312-cp312" ]; then
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH}
export PATH=/opt/_internal/cpython-3.12.0/bin/:${PATH}
#after changing "PYTHON_LIBRARY:FILEPATH" to "PYTHON_LIBRARY" ,we can use export
export DPYTHON_EXECUTABLE=/opt/conda/bin/python
export PYTHON_INCLUDE_DIR=/opt/conda/include/python3.7m
export PYTHON_LIBRARIES=/opt/conda/lib/libpython3.so
/opt/conda/bin/pip install -r ${PADDLE_ROOT}/python/requirements.txt
export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.12.0/bin/python3.12
export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.12.0/include/python3.12
export PYTHON_LIBRARIES=/opt/_internal/cpython-3.12.0/lib/libpython3.so
pip3.12 install -r ${PADDLE_ROOT}/python/requirements.txt
fi
else
pip install -r ${PADDLE_ROOT}/python/requirements.txt
Expand Down
4 changes: 2 additions & 2 deletions python/unittest_py/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ prettytable
distro
numpy>=1.20
autograd==1.4
librosa==0.8.1
librosa==0.8.1 ; python_version<"3.12"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个python3.12的时候装不了对吧

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

是的

parameterized
wandb>=0.13
wandb>=0.13 ; python_version<"3.12"
xlsxwriter==3.0.9
xdoctest==1.1.1
ubelt==1.3.3 # just for xdoctest
3 changes: 1 addition & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import sys
import time
from contextlib import contextmanager
from distutils.spawn import find_executable
from subprocess import CalledProcessError

from setuptools import Command, Extension, setup
Expand Down Expand Up @@ -58,7 +57,7 @@
)

# check cmake
CMAKE = find_executable('cmake3') or find_executable('cmake')
CMAKE = shutil.which('cmake3') or shutil.which('cmake')
assert (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这是为啥?

Copy link
Contributor Author

@tianshuo78520a tianshuo78520a Nov 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

最新版本setuptools不支持find_executable方法了,使用shutil.which进行替换,参考链接:https://peps.python.org/pep-0632/#migration-advice

CMAKE
), 'The "cmake" executable is not found. Please check if Cmake is installed.'
Expand Down
11 changes: 7 additions & 4 deletions tools/dockerfile/Dockerfile.centos
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig

RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel
RUN yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel patch
COPY build_scripts /build_scripts
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里的curl-devel-patch是干嘛的

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

安装gcc需要patch工具

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

安装gcc需要patch工具

这个是必须的吗,我记得之前也可以安装gcc啊

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

原先安装gcc没有问题,PR55835为gcc打了patch,所以需要提前安装这个工具

RUN bash build_scripts/build.sh
RUN bash build_scripts/install_nccl2.sh
#RUN bash build_scripts/install_nccl2.sh
RUN bash build_scripts/install_trt.sh
RUN rm -rf build_scripts
RUN ln -s /usr/local/ssl/include/openssl /usr/include
Expand Down Expand Up @@ -53,19 +53,22 @@ RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/re
RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install setuptools -U && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install setuptools -U && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install setuptools -U && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools -U
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools -U && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install setuptools -U

RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install -r /root/requirements.txt && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install -r /root/requirements.txt && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install -r /root/requirements.txt && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install -r /root/requirements.txt && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install -r /root/requirements.txt && \
go get github.com/Masterminds/glide && \
rm -rf /root/requirements.txt

RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.8.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.9.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0'
LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install pre-commit 'ipython==5.3.0'

RUN wget -O /opt/swig-2.0.12.tar.gz --no-check-certificate https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \
cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && cd /opt && rm swig-2.0.12.tar.gz
Expand Down
73 changes: 35 additions & 38 deletions tools/dockerfile/Dockerfile.ubuntu20
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ RUN rm /etc/apt/sources.list.d/*
RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub

RUN apt-get update --allow-unauthenticated && \
apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip \
coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx \
Expand All @@ -47,7 +48,7 @@ RUN ln -s /usr/local/gcc-12.1/bin/gcc /usr/bin/gcc
RUN ln -s /usr/local/gcc-12.1/bin/g++ /usr/bin/g++
ENV PATH=/usr/local/gcc-12.1/bin:$PATH

Run bash /build_scripts/install_cudnn.sh cudnn841
RUN bash /build_scripts/install_cudnn.sh cudnn841
ENV CUDNN_VERSION=8.4.1
#RUN bash /build_scripts/install_nccl2.sh
RUN rm -rf /build_script
Expand All @@ -58,37 +59,40 @@ RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && ta
ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH

RUN apt-get update && \
apt-get install -y python3.7 python3.7-dev python3.7-distutils\
python3.8 python3.8-dev python3.8-distutils \
apt-get install -y python3.8 python3.8-dev python3.8-distutils \
python3.9 python3.9-dev python3.9-distutils \
python3.10 python3.10-dev python3.10-distutils \
python3.11 python3.11-dev python3.11-distutils && \
python3.11 python3.11-dev python3.11-distutils \
python3.12 python3.12-dev python3.12-distutils && \
apt-get install python-is-python3 && \
rm /usr/bin/python && ln -s /usr/bin/python3.9 /usr/bin/python && \
rm /usr/bin/python3 && ln -s /usr/bin/python3.9 /usr/bin/python3

WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip
RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && unzip setuptools-50.3.2.zip
WORKDIR /home/setuptools-50.3.2
RUN python3.9 setup.py build && python3.9 setup.py install && \
python3.8 setup.py build && python3.8 setup.py install && \
python3.7 setup.py build && python3.7 setup.py install && \
python3.10 setup.py build && python3.10 setup.py install && \
python3.11 setup.py build && python3.11 setup.py install
python3.8 setup.py build && python3.8 setup.py install

WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
WORKDIR pip-20.0.1
RUN wget https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
WORKDIR /home/setuptools-68.2.2
RUN python3.10 setup.py build && python3.10 setup.py install && \
python3.11 setup.py build && python3.11 setup.py install && \
python3.12 setup.py build && python3.12 setup.py install

WORKDIR /home
RUN wget https://files.pythonhosted.org/packages/1f/7f/4da15e07ccd11c84c1ccc8f6e24288d5e76c99441bf80e315b33542db951/pip-23.3.1.tar.gz && tar -zxf pip-23.3.1.tar.gz
WORKDIR pip-23.3.1
RUN python3.9 setup.py install && \
python3.8 setup.py install && \
python3.7 setup.py install && \
python3.8 setup.py install && \
python3.10 setup.py install && \
python3.11 setup.py install
python3.11 setup.py install && \
python3.12 setup.py install

WORKDIR /home
RUN rm setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
rm -r setuptools-50.3.2 pip-20.0.1
RUN rm setuptools-50.3.2.zip setuptools-68.2.2.tar.gz pip-23.3.1.tar.gz && \
rm -r setuptools-50.3.2 setuptools-68.2.2 pip-23.3.1

# remove them when apt-get support 2.27 and higher version
RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
Expand All @@ -115,56 +119,49 @@ RUN git config --global credential.helper store
RUN localedef -i en_US -f UTF-8 en_US.UTF-8

#For pre-commit
RUN python3.7 -m pip --no-cache-dir install --upgrade pip && \
python3.8 -m pip --no-cache-dir install --upgrade pip && \
python3.9 -m pip --no-cache-dir install --upgrade pip && \
python3.10 -m pip --no-cache-dir install --upgrade pip && \
python3.11 -m pip --no-cache-dir install --upgrade pip

RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip && \
rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip3

RUN pip3.7 --no-cache-dir install ipython==5.3.0 && \
pip3.7 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.8 --no-cache-dir install ipython==5.3.0 && \
RUN pip3.8 --no-cache-dir install ipython==5.3.0 && \
pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.9 --no-cache-dir install ipython==5.3.0 && \
pip3.9 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.10 --no-cache-dir install ipython==5.3.0 && \
pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.11 --no-cache-dir install ipython==5.3.0 && \
pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel
pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel && \
pip3.12 --no-cache-dir install ipython==5.3.0 && \
pip3.12 --no-cache-dir install ipykernel==4.6.0 wheel

#For docstring checker
RUN pip3.7 --no-cache-dir install pytest astroid isort && \
pip3.8 --no-cache-dir install pytest astroid isort && \
RUN pip3.8 --no-cache-dir install pytest astroid isort && \
pip3.9 --no-cache-dir install pytest astroid isort && \
pip3.10 --no-cache-dir install pytest astroid isort && \
pip3.11 --no-cache-dir install pytest astroid isort
pip3.11 --no-cache-dir install pytest astroid isort && \
pip3.12 --no-cache-dir install pytest astroid isort

RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
RUN pip3.8 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.9 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.10 --no-cache-dir install pre-commit==2.17.0 pylint==2.12.0 && \
pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
pip3.12 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0

COPY ./python/requirements.txt /root/
COPY ./python/unittest_py/requirements.txt /home/

RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \
pip3.7 --no-cache-dir install -r /home/requirements.txt && \
pip3.8 --no-cache-dir install -r /root/requirements.txt && \
RUN pip3.8 --no-cache-dir install -r /root/requirements.txt && \
pip3.8 --no-cache-dir install -r /home/requirements.txt && \
pip3.9 --no-cache-dir install -r /root/requirements.txt && \
pip3.9 --no-cache-dir install -r /home/requirements.txt && \
pip3.10 --no-cache-dir install -r /root/requirements.txt && \
pip3.10 --no-cache-dir install -r /home/requirements.txt && \
pip3.11 --no-cache-dir install -r /root/requirements.txt && \
pip3.11 --no-cache-dir install -r /home/requirements.txt
pip3.11 --no-cache-dir install -r /home/requirements.txt && \
pip3.12 --no-cache-dir install -r /root/requirements.txt && \
pip3.12 --no-cache-dir install -r /home/requirements.txt


# ccache 4.2.0
Expand Down
3 changes: 2 additions & 1 deletion tools/dockerfile/build_scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ set -ex
# remove others to expedite build and reduce docker image size. The original
# manylinux docker image project builds many python versions.
# NOTE We added back 3.5.1, since auditwheel requires python 3.3+
CPYTHON_VERSIONS="3.11.0 3.10.0 3.9.0 3.8.0"
CPYTHON_VERSIONS="3.12.0 3.11.0 3.10.0 3.9.0 3.8.0"

# openssl version to build, with expected sha256 hash of .tar.gz
# archive
Expand Down Expand Up @@ -82,6 +82,7 @@ PY38_BIN=/opt/python/cp38-cp38/bin
PY39_BIN=/opt/python/cp39-cp39/bin
PY310_BIN=/opt/python/cp310-cp310/bin
PY311_BIN=/opt/python/cp311-cp311/bin
PY312_BIN=/opt/python/cp312-cp312/bin
# NOTE Since our custom manylinux image builds pythons with shared
# libpython, we need to add libpython's dir to LD_LIBRARY_PATH before running
# python.
Expand Down
13 changes: 7 additions & 6 deletions tools/dockerfile/build_scripts/build_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ function do_cpython_build {
rm -rf Python-$py_ver
# Some python's install as bin/python3. Make them available as
# bin/python.
if [ -e ${prefix}/bin/python3.7 ]; then
ln -s python3.7 ${prefix}/bin/python
fi
if [ -e ${prefix}/bin/python3.8 ]; then
ln -s python3.8 ${prefix}/bin/python
fi
Expand All @@ -98,16 +95,20 @@ function do_cpython_build {
if [ -e ${prefix}/bin/python3.11 ]; then
ln -s python3.11 ${prefix}/bin/python
fi
if [ -e ${prefix}/bin/python3.12 ]; then
ln -s python3.12 ${prefix}/bin/python
fi
# NOTE Make libpython shared library visible to python calls below
if [ -e ${prefix}/bin/python3.10 ] || [ -e ${prefix}/bin/python3.11 ]; then
if [ -e ${prefix}/bin/python3.10 ] || [ -e ${prefix}/bin/python3.11 ] || [ -e ${prefix}/bin/python3.12 ]; then
LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.trusted-host mirrors.aliyun.com
LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.index-url http://mirrors.aliyun.com/pypi/simple/
fi
LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py
LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.32.2
LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.40.0
cd /
ls ${MY_DIR}
local abi_tag=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
abi_version=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python -V|awk '{print $2}'|awk -F '.' '{print $1$2}')
local abi_tag=$(echo cp$abi_version-cp$abi_version)
ln -s ${prefix} /opt/python/${abi_tag}
}

Expand Down
Loading