diff --git a/.travis.yml b/.travis.yml index 5ba96c3a..66de0f51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: c os: linux -dist: bionic +dist: focal services: - docker @@ -11,31 +11,86 @@ branches: - /^pyup-/ - /^dependabot/ +cache: + directories: + - $HOME/docker + env: global: # QUAY_USERNAME and QUAY_PASSWORD for docker image upload - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw=" - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA=" +# use YAML aliases and anchors to avoid duplication in stages +# c.f. https://github.com/travis-ci/travis-ci/issues/8295#issuecomment-454457787 +_manylinux-build: &manylinux-build + stage: "Build manylinux images" + before_install: travis-ci/load_cached_images.sh combined + script: PLATFORM=$PLATFORM POLICY=$POLICY TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh glibc_skip + deploy: + provider: script + script: ./deploy.sh + on: + branch: master + repo: pypa/manylinux + jobs: include: - - arch: amd64 - env: PLATFORM="x86_64" - - arch: amd64 - env: PLATFORM="i686" - - arch: arm64 - env: PLATFORM="aarch64" - - arch: ppc64le - env: PLATFORM="ppc64le" - - arch: s390x - env: PLATFORM="s390x" - -script: - - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh - -deploy: - provider: script - script: docker/deploy.sh - on: - branch: manylinux2014 - repo: pypa/manylinux + - stage: "Build patched i686 glibc for manylinux2010_x86_64 image" + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux2010" + before_install: travis-ci/load_cached_images.sh 32 + script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh 32 + before_cache: travis-ci/cache_images.sh 32 + - stage: "Build patched x86_64 glibc for manylinux2010_x86_64 image" + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux2010" + before_install: travis-ci/load_cached_images.sh 64 + script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh 64 + before_cache: travis-ci/cache_images.sh 64 + - stage: "Add patched glibc to manylinux2010_x86_64 base image" + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux2010" + before_install: travis-ci/load_cached_images.sh all + script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh all + before_cache: travis-ci/cache_images.sh combined + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux2010" + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux2014" + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="x86_64" POLICY="manylinux_2_24" + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="i686" POLICY="manylinux2010" + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="i686" POLICY="manylinux2014" + - <<: *manylinux-build + arch: amd64 + env: PLATFORM="i686" POLICY="manylinux_2_24" + - <<: *manylinux-build + arch: arm64-graviton2 + virt: vm + group: edge + env: PLATFORM="aarch64" POLICY="manylinux2014" + - <<: *manylinux-build + arch: arm64-graviton2 + virt: vm + group: edge + env: PLATFORM="aarch64" POLICY="manylinux_2_24" + - <<: *manylinux-build + arch: ppc64le + env: PLATFORM="ppc64le" POLICY="manylinux2014" + - <<: *manylinux-build + arch: ppc64le + env: PLATFORM="ppc64le" POLICY="manylinux_2_24" + - <<: *manylinux-build + arch: s390x + env: PLATFORM="s390x" POLICY="manylinux2014" + - <<: *manylinux-build + arch: s390x + env: PLATFORM="s390x" POLICY="manylinux_2_24" diff --git a/build.sh b/build.sh index 1f39a8a7..984588e3 100755 --- a/build.sh +++ b/build.sh @@ -1,7 +1,83 @@ #!/bin/bash # Stop at any error, show all commands -set -ex +set -exuo pipefail -docker build --rm -t quay.io/pypa/manylinux2014_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/ +# rebuild glibc for manylinux2010_x86_64 +if [ "${POLICY}_${PLATFORM}" == "manylinux2010_x86_64" ] && [ "$1" != glibc_skip ] || [ "$1" == glibc_only ]; then + manylinux2010_x86_64_glibc/build.sh 32 + manylinux2010_x86_64_glibc/build.sh 64 + if [ "$1" == "glibc_only" ]; then + exit 0 + fi + manylinux2010_x86_64_glibc/build.sh all +fi + + +# Export variable needed by 'docker build --build-arg' +export POLICY +export PLATFORM + + +if [ "${PLATFORM}" == "x86_64" ]; then + MULTIARCH_PREFIX="amd64/" +elif [ "${PLATFORM}" == "i686" ]; then + MULTIARCH_PREFIX="i386/" +elif [ "${PLATFORM}" == "aarch64" ]; then + MULTIARCH_PREFIX="arm64v8/" +elif [ "${PLATFORM}" == "ppc64le" ]; then + MULTIARCH_PREFIX="ppc64le/" +elif [ "${PLATFORM}" == "s390x" ]; then + MULTIARCH_PREFIX="s390x/" +else + echo "Unsupported platform: '${PLATFORM}'" + exit 1 +fi + +if [ "${POLICY}" == "manylinux2010" ]; then + if [ "${PLATFORM}" == "x86_64" ]; then + BASEIMAGE="quay.io/pypa/manylinux2010_centos-6-no-vsyscall" + elif [ "${PLATFORM}" == "i686" ]; then + BASEIMAGE="${MULTIARCH_PREFIX}centos:6" + else + echo "Policy '${POLICY}' does not support platform '${PLATFORM}'" + exit 1 + fi + DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-8/root" + PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" + if [ "${PLATFORM}" == "i686" ]; then + LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib" + else + LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + fi +elif [ "${POLICY}" == "manylinux2014" ]; then + if [ "${PLATFORM}" == "s390x" ]; then + BASEIMAGE="s390x/clefos:7" + else + BASEIMAGE="${MULTIARCH_PREFIX}centos:7" + fi + DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-9/root" + PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" + if [ "${PLATFORM}" == "i686" ]; then + LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib" + else + LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + fi +elif [ "${POLICY}" == "manylinux_2_24" ]; then + BASEIMAGE="${MULTIARCH_PREFIX}debian:9" + DEVTOOLSET_ROOTPATH= + PREPEND_PATH= + LD_LIBRARY_PATH_ARG= +fi +export BASEIMAGE +export DEVTOOLSET_ROOTPATH +export PREPEND_PATH +export LD_LIBRARY_PATH_ARG + + +docker build \ + --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE \ + --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG \ + --rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${TRAVIS_COMMIT} \ + -f docker/Dockerfile docker/ diff --git a/docker/deploy.sh b/deploy.sh similarity index 87% rename from docker/deploy.sh rename to deploy.sh index 5d7de59a..d928d7ab 100755 --- a/docker/deploy.sh +++ b/deploy.sh @@ -1,5 +1,5 @@ #!/bin/bash -tag="quay.io/pypa/manylinux2014_$PLATFORM" +tag="quay.io/pypa/${POLICY}_${PLATFORM}" build_id=$(git show -s --format=%cd-%h --date=short $TRAVIS_COMMIT) docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..12bcaacb --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,137 @@ +# default to latest policy, x86_64 +ARG BASEIMAGE=amd64/debian:9 +ARG POLICY=manylinux_2_24 +ARG PLATFORM=x86_64 +ARG DEVTOOLSET_ROOTPATH= +ARG LD_LIBRARY_PATH_ARG= +ARG PREPEND_PATH= + + +FROM $BASEIMAGE AS runtime_base +ARG POLICY +ARG PLATFORM +ARG DEVTOOLSET_ROOTPATH +ARG LD_LIBRARY_PATH_ARG +ARG PREPEND_PATH +LABEL maintainer="The ManyLinux project" + +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} +ENV PATH=${PREPEND_PATH}${PATH} +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +# first fixup mirrors, keep the script around +COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors +RUN fixup-mirrors + +# setup entrypoint, this will wrap commands with `linux32` with i686 images +COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh +RUN /build_scripts/install-entrypoint.sh && rm -rf build_scripts +COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +ENTRYPOINT ["manylinux-entrypoint"] + +COPY build_scripts/install-runtime-packages.sh /build_scripts/install-runtime-packages.sh +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts + +COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh + +COPY build_scripts/install-autoconf.sh /build_scripts/install-autoconf.sh +RUN manylinux-entrypoint /build_scripts/install-autoconf.sh + +COPY build_scripts/install-automake.sh /build_scripts/install-automake.sh +RUN manylinux-entrypoint /build_scripts/install-automake.sh + +COPY build_scripts/install-libtool.sh /build_scripts/install-libtool.sh +RUN manylinux-entrypoint /build_scripts/install-libtool.sh + +COPY build_scripts/install-patchelf.sh /build_scripts/install-patchelf.sh +RUN manylinux-entrypoint /build_scripts/install-patchelf.sh + +COPY build_scripts/install-libxcrypt.sh /build_scripts/install-libxcrypt.sh +RUN manylinux-entrypoint /build_scripts/install-libxcrypt.sh + +CMD ["/bin/bash"] + + +FROM runtime_base AS build_base +COPY build_scripts/install-build-packages.sh /build_scripts/install-build-packages.sh +RUN manylinux-entrypoint /build_scripts/install-build-packages.sh + + +FROM build_base AS build_git +COPY build_scripts/build-git.sh /build_scripts/build-git.sh +RUN manylinux-entrypoint /build_scripts/build-git.sh + + +FROM build_base AS build_cmake +COPY build_scripts/build-cmake.sh /build_scripts/build-cmake.sh +RUN manylinux-entrypoint /build_scripts/build-cmake.sh + + +FROM build_base AS build_swig +COPY build_scripts/build-swig.sh /build_scripts/build-swig.sh +RUN manylinux-entrypoint /build_scripts/build-swig.sh + + +FROM build_base AS build_cpython +COPY build_scripts/build-sqlite3.sh /build_scripts/build-sqlite3.sh +RUN manylinux-entrypoint /build_scripts/build-sqlite3.sh + +COPY build_scripts/build-openssl.sh /build_scripts/build-openssl.sh +RUN manylinux-entrypoint /build_scripts/build-openssl.sh + +COPY build_scripts/build-cpython.sh /build_scripts/build-cpython.sh + + +FROM build_cpython AS build_cpython35 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.5.10 + +FROM build_cpython AS build_cpython36 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.12 + +FROM build_cpython AS build_cpython37 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.9 + +FROM build_cpython AS build_cpython38 +COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt +RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.6 + +FROM build_cpython AS build_cpython39 +COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt +RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.1 + +FROM runtime_base AS all_cpython +COPY --from=build_cpython35 /opt/_internal /opt/_internal/ +COPY --from=build_cpython36 /opt/_internal /opt/_internal/ +COPY --from=build_cpython37 /opt/_internal /opt/_internal/ +COPY --from=build_cpython38 /opt/_internal /opt/_internal/ +COPY --from=build_cpython39 /opt/_internal /opt/_internal/ +COPY build_scripts/install-requirements.sh /build_scripts/install-requirements.sh +COPY build_scripts/requirements.txt /build_scripts/requirements.txt +RUN manylinux-entrypoint /build_scripts/install-requirements.sh + + +FROM runtime_base +COPY --from=build_git /manylinux/git / +COPY --from=build_cmake /manylinux/cmake / +COPY --from=build_swig /manylinux/swig / +COPY --from=build_cpython /manylinux/sqlite3 / +COPY --from=all_cpython /opt/_internal /opt/_internal/ +COPY build_scripts/finalize.sh /build_scripts/finalize.sh +COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py +COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py +COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py +COPY build_scripts/requirements.txt /build_scripts/requirements.txt +COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts +ENV SSL_CERT_FILE=/opt/_internal/certs.pem diff --git a/docker/Dockerfile-aarch64 b/docker/Dockerfile-aarch64 deleted file mode 100644 index b86d6d51..00000000 --- a/docker/Dockerfile-aarch64 +++ /dev/null @@ -1,19 +0,0 @@ -FROM arm64v8/centos:7 -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_ARCH aarch64 -ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-9/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib -ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig - -COPY build_scripts /build_scripts -RUN bash build_scripts/build.sh && rm -r build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] diff --git a/docker/Dockerfile-i686 b/docker/Dockerfile-i686 deleted file mode 100644 index 16bb6e5a..00000000 --- a/docker/Dockerfile-i686 +++ /dev/null @@ -1,27 +0,0 @@ -FROM i386/centos:7 -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_ARCH i686 -ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-9/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib -ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig - -# Set a base architecture of yum package to i386 -RUN echo "i386" > /etc/yum/vars/basearch - -# To have linux32 command -RUN yum -y update && \ - yum install -y util-linux-ng - -COPY build_scripts /build_scripts -RUN linux32 bash build_scripts/build.sh && rm -r build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -ENTRYPOINT ["linux32"] -CMD ["/bin/bash"] diff --git a/docker/Dockerfile-ppc64le b/docker/Dockerfile-ppc64le deleted file mode 100644 index 43d56ec1..00000000 --- a/docker/Dockerfile-ppc64le +++ /dev/null @@ -1,19 +0,0 @@ -FROM ppc64le/centos:7 -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_ARCH ppc64le -ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-9/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib -ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig - -COPY build_scripts /build_scripts -RUN bash build_scripts/build.sh && rm -r build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] diff --git a/docker/Dockerfile-s390x b/docker/Dockerfile-s390x deleted file mode 100644 index 9e1c7799..00000000 --- a/docker/Dockerfile-s390x +++ /dev/null @@ -1,19 +0,0 @@ -FROM s390x/clefos:7 -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_ARCH s390x -ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-9/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib -ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig - -COPY build_scripts /build_scripts -RUN bash build_scripts/build.sh && rm -r build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] diff --git a/docker/Dockerfile-x86_64 b/docker/Dockerfile-x86_64 deleted file mode 100644 index 225b4aaf..00000000 --- a/docker/Dockerfile-x86_64 +++ /dev/null @@ -1,19 +0,0 @@ -FROM centos:7 -LABEL maintainer="The ManyLinux project" - -ENV AUDITWHEEL_ARCH x86_64 -ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-9/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib -ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig - -COPY build_scripts /build_scripts -RUN bash build_scripts/build.sh && rm -r build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] diff --git a/docker/build_scripts/build-cmake.sh b/docker/build_scripts/build-cmake.sh new file mode 100755 index 00000000..273284e8 --- /dev/null +++ b/docker/build_scripts/build-cmake.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install newest cmake +CMAKE_VERSION=3.18.4 +CMAKE_HASH=597c61358e6a92ecbfad42a9b5321ddd801fc7e7eca08441307c9138382d4f77 +CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download + + +fetch_source cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_DOWNLOAD_URL}/v${CMAKE_VERSION} +check_sha256sum cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_HASH} +tar -xzf cmake-${CMAKE_VERSION}.tar.gz +pushd cmake-${CMAKE_VERSION} +./bootstrap --system-curl --parallel=$(nproc) +make -j$(nproc) +make install DESTDIR=/manylinux/cmake +popd +rm -rf cmake-${CMAKE_VERSION} + +# remove help +rm -rf /manylinux/cmake/usr/local/share/cmake-*/Help + +# Strip what we can +strip_ /manylinux/cmake + +# Install +cp -rf /manylinux/cmake/* / + + +cmake --version diff --git a/docker/build_scripts/build-cpython.sh b/docker/build_scripts/build-cpython.sh new file mode 100755 index 00000000..24f948b3 --- /dev/null +++ b/docker/build_scripts/build-cpython.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + + +CPYTHON_VERSION=$1 +CPYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python + + +function pyver_dist_dir { + # Echoes the dist directory name of given pyver, removing alpha/beta prerelease + # Thus: + # 3.2.1 -> 3.2.1 + # 3.7.0b4 -> 3.7.0 + echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}' +} + + +CPYTHON_DIST_DIR=$(pyver_dist_dir ${CPYTHON_VERSION}) +fetch_source Python-${CPYTHON_VERSION}.tgz ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR} +fetch_source Python-${CPYTHON_VERSION}.tgz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR} +gpg --verify Python-${CPYTHON_VERSION}.tgz.asc +tar -xzf Python-${CPYTHON_VERSION}.tgz +pushd Python-${CPYTHON_VERSION} +PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}" +mkdir -p ${PREFIX}/lib +./configure --prefix=${PREFIX} --disable-shared > /dev/null +make -j$(nproc) > /dev/null +make -j$(nproc) install > /dev/null +popd +rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc + +# we don't need libpython*.a, and they're many megabytes +find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f + +# We do not need the Python test suites +find ${PREFIX} -depth \( -type d -a -name test -o -name tests \) | xargs rm -rf + +# Strip ELF files found in ${PREFIX} +strip_ ${PREFIX} + +# Some python's install as bin/python3. Make them available as bin/python. +if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then + ln -s python3 ${PREFIX}/bin/python +fi +${PREFIX}/bin/python -m ensurepip +if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then + ln -s pip3 ${PREFIX}/bin/pip +fi + +# We do not need precompiled .pyc and .pyo files. +clean_pyc ${PREFIX} diff --git a/docker/build_scripts/build-git.sh b/docker/build_scripts/build-git.sh new file mode 100755 index 00000000..ba719635 --- /dev/null +++ b/docker/build_scripts/build-git.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install newest libtool +GIT_ROOT=git-2.29.2 +GIT_HASH=869a121e1d75e4c28213df03d204156a17f02fce2dc77be9795b327830f54195 +GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git + + +fetch_source ${GIT_ROOT}.tar.gz ${GIT_DOWNLOAD_URL} +check_sha256sum ${GIT_ROOT}.tar.gz ${GIT_HASH} +tar -xzf ${GIT_ROOT}.tar.gz +pushd ${GIT_ROOT} +make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux/git +popd +rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz + + +# Strip what we can +strip_ /manylinux/git + +# Install +cp -rf /manylinux/git/* / + +git version diff --git a/docker/build_scripts/build-openssl.sh b/docker/build_scripts/build-openssl.sh new file mode 100755 index 00000000..eeab6599 --- /dev/null +++ b/docker/build_scripts/build-openssl.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install a more recent openssl +OPENSSL_MIN_VERSION=1.0.2 +OPENSSL_VERSION=1.1.1i +OPENSSL_ROOT=openssl-${OPENSSL_VERSION} +OPENSSL_HASH=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source + + +INSTALLED=$(openssl version | head -1 | awk '{ print $2 }') +SMALLEST=$(echo -e "${INSTALLED}\n${OPENSSL_MIN_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1) +if [ "${SMALLEST}" == "${OPENSSL_MIN_VERSION}" ]; then + echo "skipping installation of openssl ${OPENSSL_VERSION}, system provides openssl ${INSTALLED} which is newer than openssl ${OPENSSL_MIN_VERSION}" + exit 0 +fi + +if which yum; then + yum erase -y openssl-devel +else + apt-get remove -y libssl-dev +fi + +fetch_source ${OPENSSL_ROOT}.tar.gz ${OPENSSL_DOWNLOAD_URL} +check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} +tar -xzf ${OPENSSL_ROOT}.tar.gz +pushd ${OPENSSL_ROOT} +./config no-shared -fPIC --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null +make > /dev/null +make install_sw > /dev/null +popd +rm -rf ${OPENSSL_ROOT} ${OPENSSL_ROOT}.tar.gz + + +/usr/local/ssl/bin/openssl version diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh new file mode 100755 index 00000000..0bebcb9c --- /dev/null +++ b/docker/build_scripts/build-sqlite3.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install a more recent SQLite3 +SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3330000 +SQLITE_AUTOCONF_HASH=106a2c48c7f75a298a7557bcc0d5f4f454e5b43811cc738b7ca294d6956bbb15 +SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 + + +fetch_source ${SQLITE_AUTOCONF_VERSION}.tar.gz ${SQLITE_AUTOCONF_DOWNLOAD_URL} +check_sha256sum ${SQLITE_AUTOCONF_VERSION}.tar.gz ${SQLITE_AUTOCONF_HASH} +tar xfz ${SQLITE_AUTOCONF_VERSION}.tar.gz +pushd ${SQLITE_AUTOCONF_VERSION} +DESTDIR=/manylinux/sqlite3 do_standard_install +popd +rm -rf ${SQLITE_AUTOCONF_VERSION} ${SQLITE_AUTOCONF_VERSION}.tar.gz + +# static library is unused, remove it +rm /manylinux/sqlite3/usr/local/lib/libsqlite3.a + +# Install +cp -rf /manylinux/sqlite3/* / + +# Clean-up for runtime +rm -rf /manylinux/sqlite3/usr/local/bin /manylinux/sqlite3/usr/local/include /manylinux/sqlite3/usr/local/lib/pkg-config /manylinux/sqlite3/usr/local/share +find -L manylinux/sqlite3 -type f -a -not -name 'libsqlite3.so.*' -delete diff --git a/docker/build_scripts/build-swig.sh b/docker/build_scripts/build-swig.sh new file mode 100755 index 00000000..8cfdc7d1 --- /dev/null +++ b/docker/build_scripts/build-swig.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install newest swig +SWIG_VERSION=4.0.2 +SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc +SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/swig-${SWIG_VERSION} + +PCRE_VERSION=8.44 +PCRE_HASH=aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728 +PCRE_DOWNLOAD_URL=https://ftp.pcre.org/pub/pcre + +fetch_source swig-${SWIG_VERSION}.tar.gz ${SWIG_DOWNLOAD_URL}/ +check_sha256sum swig-${SWIG_VERSION}.tar.gz ${SWIG_HASH} +tar -xzf swig-${SWIG_VERSION}.tar.gz +pushd swig-${SWIG_VERSION} +fetch_source pcre-${PCRE_VERSION}.tar.gz ${PCRE_DOWNLOAD_URL}/ +check_sha256sum pcre-${PCRE_VERSION}.tar.gz ${PCRE_HASH} +./Tools/pcre-build.sh +./configure +make -j$(nproc) +make install DESTDIR=/manylinux/swig +popd +rm -rf swig-${SWIG_VERSION} + +# Strip what we can +strip_ /manylinux/swig + +# Install +cp -rf /manylinux/swig/* / + + +swig -version diff --git a/docker/build_scripts/build.sh b/docker/build_scripts/build.sh deleted file mode 100755 index 320758e2..00000000 --- a/docker/build_scripts/build.sh +++ /dev/null @@ -1,235 +0,0 @@ -#!/bin/bash -# Top-level build script called from Dockerfile - -# Stop at any error, show all commands -set -ex - -# Set build environment variables -MY_DIR=$(dirname "${BASH_SOURCE[0]}") -. $MY_DIR/build_env.sh - -# Dependencies for compiling Python that we want to remove from -# the final image after compiling Python -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel" - -# Libraries that are allowed as part of the manylinux2014 profile -# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy -# On RPM-based systems, they are provided by these packages: -# Package: Libraries -# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 -# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 -# libICE: libICE.so.6 -# libX11: libX11.so.6 -# libXext: libXext.so.6 -# libXrender: libXrender.so.1 -# libgcc: libgcc_s.so.1 -# libstdc++: libstdc++.so.6 -# mesa: libGL.so.1 -# -# PEP is missing the package for libSM.so.6 for RPM based system -# Install development packages (except for libgcc which is provided by gcc install) -MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" - -CMAKE_DEPS="openssl-devel zlib-devel libcurl-devel" - -# Get build utilities -source $MY_DIR/build_utils.sh - -# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages -echo "multilib_policy=best" >> /etc/yum.conf -# Error out if requested packages do not exist -echo "skip_missing_names_on_install=False" >> /etc/yum.conf -# Make sure that locale will not be removed -sed -i '/^override_install_langs=/d' /etc/yum.conf - -# https://hub.docker.com/_/centos/ -# "Additionally, images with minor version tags that correspond to install -# media are also offered. These images DO NOT recieve updates as they are -# intended to match installation iso contents. If you choose to use these -# images it is highly recommended that you include RUN yum -y update && yum -# clean all in your Dockerfile, or otherwise address any potential security -# concerns." -# Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129 - -if [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then - # workaround for https://github.com/nealef/clefos/issues/5 - # this shall be removed ASAP - yum -y install epel-release-7-12 - yum -y --exclude=epel-release update -else - yum -y update -fi -yum -y install yum-utils curl -yum-config-manager --enable extras - -if ! which localedef &> /dev/null; then - # somebody messed up glibc-common package to squeeze image size, reinstall the package - yum -y reinstall glibc-common -fi - -# upgrading glibc-common can end with removal on en_US.UTF-8 locale -localedef -i en_US -f UTF-8 en_US.UTF-8 - -TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" -if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh - # EPEL support (for yasm) - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - YASM=yasm -elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then - # Software collection (for devtoolset-9) - yum -y install centos-release-scl-rh -elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then - # No yasm on i686 - # Install mayeut/devtoolset-9 repo to get devtoolset-9 - curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo -fi - -# Development tools and libraries -yum -y install \ - autoconf \ - automake \ - bison \ - bzip2 \ - ${TOOLCHAIN_DEPS} \ - diffutils \ - gettext \ - file \ - kernel-devel \ - libffi-devel \ - make \ - patch \ - unzip \ - which \ - ${YASM} \ - ${PYTHON_COMPILE_DEPS} \ - ${CMAKE_DEPS} - -# Install git -build_git $GIT_ROOT $GIT_HASH -git version - -# Install newest automake -build_automake $AUTOMAKE_ROOT $AUTOMAKE_HASH -automake --version - -# Install newest libtool -build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH -libtool --version - -# Install a more recent SQLite3 -curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_VERSION.tar.gz -check_sha256sum $SQLITE_AUTOCONF_VERSION.tar.gz $SQLITE_AUTOCONF_HASH -tar xfz $SQLITE_AUTOCONF_VERSION.tar.gz -cd $SQLITE_AUTOCONF_VERSION -do_standard_install -cd .. -rm -rf $SQLITE_AUTOCONF_VERSION* -rm /usr/local/lib/libsqlite3.a - -# Install a recent version of cmake3 -curl -L -O $CMAKE_DOWNLOAD_URL/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -check_sha256sum cmake-${CMAKE_VERSION}.tar.gz $CMAKE_HASH -tar -xzf cmake-${CMAKE_VERSION}.tar.gz -cd cmake-${CMAKE_VERSION} -./bootstrap --system-curl --parallel=$(nproc) -make -j$(nproc) -make install -cd .. -rm -rf cmake-${CMAKE_VERSION} - -# Install libcrypt.so.1 and libcrypt.so.2 -build_libxcrypt "$LIBXCRYPT_DOWNLOAD_URL" "$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" - -# Compile the latest Python releases. -# (In order to have a proper SSL module, Python is compiled -# against a recent openssl [see env vars above], which is linked -# statically. -mkdir -p /opt/python -build_cpythons $CPYTHON_VERSIONS - -# Create venv for auditwheel & certifi -TOOLS_PATH=/opt/_internal/tools -/opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH -source $TOOLS_PATH/bin/activate - -# Install default packages -pip install -U --require-hashes -r $MY_DIR/requirements.txt -# Install certifi and auditwheel -pip install -U --require-hashes -r $MY_DIR/requirements-tools.txt - -# Make auditwheel available in PATH -ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel - -# Our openssl doesn't know how to find the system CA trust store -# (https://github.com/pypa/manylinux/issues/53) -# And it's not clear how up-to-date that is anyway -# So let's just use the same one pip and everyone uses -ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem -# If you modify this line you also have to modify the versions in the Dockerfiles: -export SSL_CERT_FILE=/opt/_internal/certs.pem - -# Deactivate the tools virtual environment -deactivate - -# Install patchelf (latest with unreleased bug fixes) and apply our patches -build_patchelf $PATCHELF_VERSION $PATCHELF_HASH - -# Clean up development headers and other unnecessary stuff for -# final image -yum -y erase \ - avahi \ - bitstream-vera-fonts \ - freetype \ - gettext \ - gtk2 \ - hicolor-icon-theme \ - libX11 \ - wireless-tools \ - ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 -yum -y install ${MANYLINUX_DEPS} -yum -y clean all > /dev/null 2>&1 -yum list installed - -# we don't need libpython*.a, and they're many megabytes -find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f - -# Strip what we can -- and ignore errors, because this just attempts to strip -# *everything*, including non-ELF files: -find /opt/_internal -type f -print0 \ - | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true -find /usr/local -type f -print0 \ - | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true - -for PYTHON in /opt/python/*/bin/python; do - # Smoke test to make sure that our Pythons work, and do indeed detect as - # being manylinux compatible: - $PYTHON $MY_DIR/manylinux-check.py - # Make sure that SSL cert checking works - $PYTHON $MY_DIR/ssl-check.py -done - -# We do not need the Python test suites, or indeed the precompiled .pyc and -# .pyo files. Partially cribbed from: -# https://github.com/docker-library/python/blob/master/3.4/slim/Dockerfile -find /opt/_internal -depth \ - \( -type d -a -name test -o -name tests \) \ - -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) | xargs rm -rf - -# Fix libc headers to remain compatible with C99 compilers. -find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + - -if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then - # remove useless things that have been installed by devtoolset - rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man - find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -fi -rm -rf /usr/share/backgrounds -# if we updated glibc, we need to strip locales again... -localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive -mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl -build-locale-archive -find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf -rm -rf /usr/local/share/man diff --git a/docker/build_scripts/build_env.sh b/docker/build_scripts/build_env.sh deleted file mode 100644 index c172ca31..00000000 --- a/docker/build_scripts/build_env.sh +++ /dev/null @@ -1,32 +0,0 @@ -# source me - -PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python -# of the form .. or ..rc -CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 3.9.1" - -PATCHELF_VERSION=0.12 -PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 - -AUTOMAKE_ROOT=automake-1.16.2 -AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 -AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake - -LIBTOOL_ROOT=libtool-2.4.6 -LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 -LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool - -SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3330000 -SQLITE_AUTOCONF_HASH=106a2c48c7f75a298a7557bcc0d5f4f454e5b43811cc738b7ca294d6956bbb15 -SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 - -CMAKE_VERSION=3.18.3 -CMAKE_HASH=2c89f4e30af4914fd6fb5d00f863629812ada848eee4e2d29ec7e456d7fa32e5 -CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download - -LIBXCRYPT_VERSION=4.4.17 -LIBXCRYPT_DOWNLOAD_URL=https://codeload.github.com/besser82/libxcrypt/tar.gz -LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 - -GIT_ROOT=git-2.29.1 -GIT_HASH=728845a66103d8d1572a656123c2c09d7aa4c0ab8f4c3dc3911e14e18c85ee67 -GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 5a36434e..c9ba3daf 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -3,212 +3,54 @@ function check_var { - if [ -z "$1" ]; then - echo "required variable not defined" - exit 1 - fi -} - - -function lex_pyver { - # Echoes Python version string padded with zeros - # Thus: - # 3.2.1 -> 003002001 - # 3 -> 003000000 - echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' -} - - -function pyver_dist_dir { - # Echoes the dist directory name of given pyver, removing alpha/beta prerelease - # Thus: - # 3.2.1 -> 3.2.1 - # 3.7.0b4 -> 3.7.0 - echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}' -} - - -function do_cpython_build { - local py_ver=$1 - check_var $py_ver - tar -xzf Python-$py_ver.tgz - pushd Python-$py_ver - local prefix="/opt/_internal/cpython-${py_ver}" - mkdir -p ${prefix}/lib - ./configure --prefix=${prefix} --disable-shared > /dev/null - make -j$(nproc) > /dev/null - make -j$(nproc) install > /dev/null - popd - rm -rf Python-$py_ver - # Some python's install as bin/python3. Make them available as - # bin/python. - if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then - ln -s python3 ${prefix}/bin/python - fi - ${prefix}/bin/python -m ensurepip - if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then - ln -s pip3 ${prefix}/bin/pip - fi - # Since we fall back on a canned copy of pip, we might not have - # the latest pip and friends. Upgrade them to make sure. - ${prefix}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt - local abi_tag=$(${prefix}/bin/python ${MY_DIR}/python-tag-abi-tag.py) - ln -s ${prefix} /opt/python/${abi_tag} -} - - -function build_cpython { - local py_ver=$1 - check_var $py_ver - check_var $PYTHON_DOWNLOAD_URL - local py_dist_dir=$(pyver_dist_dir $py_ver) - curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz - curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz.asc - gpg --verify Python-$py_ver.tgz.asc - do_cpython_build $py_ver - rm -f Python-$py_ver.tgz - rm -f Python-$py_ver.tgz.asc -} - - -function build_cpythons { - # Import public keys used to verify downloaded Python source tarballs. - # https://www.python.org/static/files/pubkeys.txt - gpg --import ${MY_DIR}/cpython-pubkeys.txt - # Add version 3.8, 3.9 release manager's key - gpg --import ${MY_DIR}/ambv-pubkey.txt - for py_ver in $@; do - build_cpython $py_ver - done - # Remove GPG hidden directory. - rm -rf /root/.gnupg/ + if [ -z "$1" ]; then + echo "required variable not defined" + exit 1 + fi } function fetch_source { - # This is called both inside and outside the build context (e.g. in Travis) to prefetch - # source tarballs, where curl exists (and works) - local file=$1 - check_var ${file} - local url=$2 - check_var ${url} - if [ -f ${file} ]; then - echo "${file} exists, skipping fetch" - else - curl -fsSL -o ${file} ${url}/${file} - fi + # This is called both inside and outside the build context (e.g. in Travis) to prefetch + # source tarballs, where curl exists (and works) + local file=$1 + check_var ${file} + local url=$2 + check_var ${url} + if [ -f ${file} ]; then + echo "${file} exists, skipping fetch" + else + curl -fsSL -o ${file} ${url}/${file} + fi } function check_sha256sum { - local fname=$1 - check_var ${fname} - local sha256=$2 - check_var ${sha256} + local fname=$1 + check_var ${fname} + local sha256=$2 + check_var ${sha256} - echo "${sha256} ${fname}" > ${fname}.sha256 - sha256sum -c ${fname}.sha256 - rm -f ${fname}.sha256 -} - - -function build_git { - local git_fname=$1 - check_var ${git_fname} - local git_sha256=$2 - check_var ${git_sha256} - check_var ${GIT_DOWNLOAD_URL} - fetch_source ${git_fname}.tar.gz ${GIT_DOWNLOAD_URL} - check_sha256sum ${git_fname}.tar.gz ${git_sha256} - tar -xzf ${git_fname}.tar.gz - (cd ${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null) - rm -rf ${git_fname} ${git_fname}.tar.gz + echo "${sha256} ${fname}" > ${fname}.sha256 + sha256sum -c ${fname}.sha256 + rm -f ${fname}.sha256 } function do_standard_install { - ./configure "$@" > /dev/null - make -j$(nproc) > /dev/null - make -j$(nproc) install > /dev/null + ./configure "$@" > /dev/null + make -j$(nproc) > /dev/null + make -j$(nproc) install-strip > /dev/null } -function build_autoconf { - local autoconf_fname=$1 - check_var ${autoconf_fname} - local autoconf_sha256=$2 - check_var ${autoconf_sha256} - check_var ${AUTOCONF_DOWNLOAD_URL} - fetch_source ${autoconf_fname}.tar.gz ${AUTOCONF_DOWNLOAD_URL} - check_sha256sum ${autoconf_fname}.tar.gz ${autoconf_sha256} - tar -zxf ${autoconf_fname}.tar.gz - (cd ${autoconf_fname} && do_standard_install) - rm -rf ${autoconf_fname} ${autoconf_fname}.tar.gz +function clean_pyc { + find $1 -type f -a \( -name '*.pyc' -o -name '*.pyo' \) -delete } -function build_automake { - local automake_fname=$1 - check_var ${automake_fname} - local automake_sha256=$2 - check_var ${automake_sha256} - check_var ${AUTOMAKE_DOWNLOAD_URL} - fetch_source ${automake_fname}.tar.gz ${AUTOMAKE_DOWNLOAD_URL} - check_sha256sum ${automake_fname}.tar.gz ${automake_sha256} - tar -zxf ${automake_fname}.tar.gz - (cd ${automake_fname} && do_standard_install) - rm -rf ${automake_fname} ${automake_fname}.tar.gz -} - - -function build_libtool { - local libtool_fname=$1 - check_var ${libtool_fname} - local libtool_sha256=$2 - check_var ${libtool_sha256} - check_var ${LIBTOOL_DOWNLOAD_URL} - fetch_source ${libtool_fname}.tar.gz ${LIBTOOL_DOWNLOAD_URL} - check_sha256sum ${libtool_fname}.tar.gz ${libtool_sha256} - tar -zxf ${libtool_fname}.tar.gz - (cd ${libtool_fname} && do_standard_install) - rm -rf ${libtool_fname} ${libtool_fname}.tar.gz -} - -function build_libxcrypt { - curl -fsSLO "$LIBXCRYPT_DOWNLOAD_URL"/v"$LIBXCRYPT_VERSION" - check_sha256sum "v$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" - tar xfz "v$LIBXCRYPT_VERSION" - pushd "libxcrypt-$LIBXCRYPT_VERSION" - ./autogen.sh > /dev/null - do_standard_install \ - --disable-obsolete-api \ - --enable-hashes=all \ - --disable-werror - # we also need libcrypt.so.1 with glibc compatibility for system libraries - # c.f https://github.com/pypa/manylinux/issues/305#issuecomment-625902928 - make clean > /dev/null - sed -r -i 's/XCRYPT_([0-9.])+/-/g;s/(%chain OW_CRYPT_1.0).*/\1/g' lib/libcrypt.map.in - DESTDIR=$(pwd)/so.1 do_standard_install \ - --disable-xcrypt-compat-files \ - --enable-obsolete-api=glibc \ - --enable-hashes=all \ - --disable-werror - cp -P ./so.1/usr/local/lib/libcrypt.so.1* /usr/local/lib/ - popd - rm -rf "v$LIBXCRYPT_VERSION" "libxcrypt-$LIBXCRYPT_VERSION" - - # Delete GLIBC version headers and libraries - rm -rf /usr/include/crypt.h - rm -rf /usr/lib*/libcrypt.a /usr/lib*/libcrypt.so /usr/lib*/libcrypt.so.1 -} - -function build_patchelf { - local patchelf_version=$1 - local patchelf_hash=$2 - local src_dir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) - curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$patchelf_version.tar.gz - check_sha256sum patchelf.tar.gz $patchelf_hash - tar -xzf patchelf.tar.gz - (cd patchelf-$patchelf_version && ./bootstrap.sh && do_standard_install) - rm -rf patchelf.tar.gz patchelf-$patchelf_version +function strip_ { + # Strip what we can -- and ignore errors, because this just attempts to strip + # *everything*, including non-ELF files: + find $1 -type f -print0 | xargs -0 -n1 strip --strip-unneeded 2>/dev/null || true } diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh new file mode 100755 index 00000000..41df55fe --- /dev/null +++ b/docker/build_scripts/finalize.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + + +mkdir /opt/python +for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 -name 'cpython*'); do + ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py) + ln -s ${PREFIX} /opt/python/${ABI_TAG} +done + +# Create venv for auditwheel & certifi +TOOLS_PATH=/opt/_internal/tools +/opt/python/cp38-cp38/bin/python -m venv $TOOLS_PATH +source $TOOLS_PATH/bin/activate + +# Install default packages +pip install --no-cache-dir -U --require-hashes -r $MY_DIR/requirements.txt +# Install certifi and auditwheel +pip install --no-cache-dir -U --require-hashes -r $MY_DIR/requirements-tools.txt + +# Make auditwheel available in PATH +ln -s $TOOLS_PATH/bin/auditwheel /usr/local/bin/auditwheel + +# Our openssl doesn't know how to find the system CA trust store +# (https://github.com/pypa/manylinux/issues/53) +# And it's not clear how up-to-date that is anyway +# So let's just use the same one pip and everyone uses +ln -s $(python -c 'import certifi; print(certifi.where())') /opt/_internal/certs.pem +# If you modify this line you also have to modify the versions in the Dockerfiles: +export SSL_CERT_FILE=/opt/_internal/certs.pem + +# Uninstall pip which is no longer required +python -m pip uninstall -y pip + +# Deactivate the tools virtual environment +deactivate + + +for PYTHON in /opt/python/*/bin/python; do + # Smoke test to make sure that our Pythons work, and do indeed detect as + # being manylinux compatible: + $PYTHON $MY_DIR/manylinux-check.py ${AUDITWHEEL_POLICY} ${AUDITWHEEL_ARCH} + # Make sure that SSL cert checking works + $PYTHON $MY_DIR/ssl-check.py +done + +clean_pyc /opt/_internal diff --git a/docker/build_scripts/fixup-mirrors.sh b/docker/build_scripts/fixup-mirrors.sh new file mode 100755 index 00000000..920f078b --- /dev/null +++ b/docker/build_scripts/fixup-mirrors.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Fix up mirrors once distro reaches EOL + +# Stop at any error, show all commands +set -exuo pipefail + +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + # Centos 6 is EOL and is no longer available from the usual mirrors, so switch + # to https://vault.centos.org + sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf + sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/*.repo + sed -i 's;^.*baseurl=http://mirror;baseurl=https://vault;g' /etc/yum.repos.d/*.repo + sed -i 's;^.*baseurl=http://download.fedoraproject.org/pub;baseurl=https://archives.fedoraproject.org/pub/archive;g' /etc/yum.repos.d/*.repo +fi diff --git a/docker/build_scripts/install-autoconf.sh b/docker/build_scripts/install-autoconf.sh new file mode 100755 index 00000000..c97fb054 --- /dev/null +++ b/docker/build_scripts/install-autoconf.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + + +# Install newest autoconf +AUTOCONF_VERSION=2.69 +AUTOCONF_ROOT=autoconf-${AUTOCONF_VERSION} +AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 +AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf + + +if autoconf --version > /dev/null 2>&1; then + INSTALLED=$(autoconf --version | head -1 | awk '{ print $NF }') + SMALLEST=$(echo -e "${INSTALLED}\n${AUTOCONF_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1) + if [ "${SMALLEST}" == "${AUTOCONF_VERSION}" ]; then + echo "skipping installation of autoconf ${AUTOCONF_VERSION}, system provides autoconf ${INSTALLED}" + exit 0 + fi +fi + + +fetch_source ${AUTOCONF_ROOT}.tar.gz ${AUTOCONF_DOWNLOAD_URL} +check_sha256sum ${AUTOCONF_ROOT}.tar.gz ${AUTOCONF_HASH} +tar -zxf ${AUTOCONF_ROOT}.tar.gz +pushd ${AUTOCONF_ROOT} +do_standard_install +popd +rm -rf ${AUTOCONF_ROOT} ${AUTOCONF_ROOT}.tar.gz + + +hash -r +autoconf --version diff --git a/docker/build_scripts/install-automake.sh b/docker/build_scripts/install-automake.sh new file mode 100755 index 00000000..ee2e50d8 --- /dev/null +++ b/docker/build_scripts/install-automake.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install newest automake +AUTOMAKE_VERSION=1.16.2 +AUTOMAKE_ROOT=automake-${AUTOMAKE_VERSION} +AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 +AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake + + +if automake --version > /dev/null 2>&1; then + INSTALLED=$(automake --version | head -1 | awk '{ print $NF }') + SMALLEST=$(echo -e "${INSTALLED}\n${AUTOMAKE_VERSION}" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -1) + if [ "${SMALLEST}" == "${AUTOMAKE_VERSION}" ]; then + echo "skipping installation of automake ${AUTOMAKE_VERSION}, system provides automake ${INSTALLED}" + exit 0 + fi +fi + + +fetch_source ${AUTOMAKE_ROOT}.tar.gz ${AUTOMAKE_DOWNLOAD_URL} +check_sha256sum ${AUTOMAKE_ROOT}.tar.gz ${AUTOMAKE_HASH} +tar -zxf ${AUTOMAKE_ROOT}.tar.gz +pushd ${AUTOMAKE_ROOT} +do_standard_install +popd +rm -rf ${AUTOMAKE_ROOT} ${AUTOMAKE_ROOT}.tar.gz + + +hash -r +automake --version diff --git a/docker/build_scripts/install-build-packages.sh b/docker/build_scripts/install-build-packages.sh new file mode 100755 index 00000000..9d20ce7b --- /dev/null +++ b/docker/build_scripts/install-build-packages.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Install packages that will be needed at runtime + +# Stop at any error, show all commands +set -exuo pipefail + + +# if a devel package is added to COMPILE_DEPS, +# make sure the corresponding library is added to RUNTIME_DEPS if applicable + +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + PACKAGE_MANAGER=yum + COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel uuid-devel libffi-devel kernel-headers" + if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + COMPILE_DEPS="${COMPILE_DEPS} db4-devel" + else + COMPILE_DEPS="${COMPILE_DEPS} libdb-devel" + fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + PACKAGE_MANAGER=apt + COMPILE_DEPS="libz-dev libbz2-dev libexpat1-dev libncurses5-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-kernel-headers" +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + + +if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y install ${COMPILE_DEPS} + yum clean all + rm -rf /var/cache/yum +elif [ ${PACKAGE_MANAGER} == apt ]; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -qq -y --no-install-recommends ${COMPILE_DEPS} + apt-get clean -qq + rm -rf /var/lib/apt/lists/* +else + echo "Not implemented" + exit 1 +fi diff --git a/docker/build_scripts/install-entrypoint.sh b/docker/build_scripts/install-entrypoint.sh new file mode 100755 index 00000000..7c340f08 --- /dev/null +++ b/docker/build_scripts/install-entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Install entrypoint: +# make sure yum is configured correctly and linux32 is available on i686 + +# Stop at any error, show all commands +set -exuo pipefail + +if [ "${AUDITWHEEL_PLAT}" == "manylinux2010_i686" ] || [ "${AUDITWHEEL_PLAT}" == "manylinux2014_i686" ]; then + echo "i386" > /etc/yum/vars/basearch + yum -y update + yum install -y util-linux-ng + yum clean all + rm -rf /var/cache/yum +fi diff --git a/docker/build_scripts/install-libtool.sh b/docker/build_scripts/install-libtool.sh new file mode 100755 index 00000000..82b0d2f5 --- /dev/null +++ b/docker/build_scripts/install-libtool.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install newest libtool +LIBTOOL_VERSION=2.4.6 +LIBTOOL_ROOT=libtool-${LIBTOOL_VERSION} +LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 +LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool + + +fetch_source ${LIBTOOL_ROOT}.tar.gz ${LIBTOOL_DOWNLOAD_URL} +check_sha256sum ${LIBTOOL_ROOT}.tar.gz ${LIBTOOL_HASH} +tar -zxf ${LIBTOOL_ROOT}.tar.gz +pushd ${LIBTOOL_ROOT} +do_standard_install +popd +rm -rf ${LIBTOOL_ROOT} ${LIBTOOL_ROOT}.tar.gz + + +hash -r +libtoolize --version diff --git a/docker/build_scripts/install-libxcrypt.sh b/docker/build_scripts/install-libxcrypt.sh new file mode 100755 index 00000000..cd4e40c5 --- /dev/null +++ b/docker/build_scripts/install-libxcrypt.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install libcrypt.so.1 and libcrypt.so.2 +LIBXCRYPT_VERSION=4.4.17 +LIBXCRYPT_DOWNLOAD_URL=https://codeload.github.com/besser82/libxcrypt/tar.gz +LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 + + +curl -fsSLO "$LIBXCRYPT_DOWNLOAD_URL"/v"$LIBXCRYPT_VERSION" +check_sha256sum "v$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" +tar xfz "v$LIBXCRYPT_VERSION" +pushd "libxcrypt-$LIBXCRYPT_VERSION" +./autogen.sh > /dev/null +do_standard_install \ + --disable-obsolete-api \ + --enable-hashes=all \ + --disable-werror +# we also need libcrypt.so.1 with glibc compatibility for system libraries +# c.f https://github.com/pypa/manylinux/issues/305#issuecomment-625902928 +make clean > /dev/null +sed -r -i 's/XCRYPT_([0-9.])+/-/g;s/(%chain OW_CRYPT_1.0).*/\1/g' lib/libcrypt.map.in +DESTDIR=$(pwd)/so.1 do_standard_install \ + --disable-xcrypt-compat-files \ + --enable-obsolete-api=glibc \ + --enable-hashes=all \ + --disable-werror +cp -P ./so.1/usr/local/lib/libcrypt.so.1* /usr/local/lib/ +popd +rm -rf "v$LIBXCRYPT_VERSION" "libxcrypt-$LIBXCRYPT_VERSION" + +# Delete GLIBC version headers and libraries +rm -rf /usr/include/crypt.h +rm -rf /usr/lib*/libcrypt.a /usr/lib*/libcrypt.so /usr/lib*/libcrypt.so.1 diff --git a/docker/build_scripts/install-patchelf.sh b/docker/build_scripts/install-patchelf.sh new file mode 100755 index 00000000..32f1e916 --- /dev/null +++ b/docker/build_scripts/install-patchelf.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + +# Install patchelf (latest with unreleased bug fixes) and apply our patches +PATCHELF_VERSION=0.12 +PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 + + +curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz +check_sha256sum patchelf.tar.gz $PATCHELF_HASH +tar -xzf patchelf.tar.gz +pushd patchelf-$PATCHELF_VERSION +./bootstrap.sh +do_standard_install +popd +rm -rf patchelf.tar.gz patchelf-$PATCHELF_VERSION + + +patchelf --version diff --git a/docker/build_scripts/install-requirements.sh b/docker/build_scripts/install-requirements.sh new file mode 100755 index 00000000..8138f244 --- /dev/null +++ b/docker/build_scripts/install-requirements.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Top-level build script called from Dockerfile + +# Stop at any error, show all commands +set -exuo pipefail + +# Get script directory +MY_DIR=$(dirname "${BASH_SOURCE[0]}") + +# Get build utilities +source $MY_DIR/build_utils.sh + + +for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 -name 'cpython*'); do + # Since we fall back on a canned copy of pip, we might not have + # the latest pip and friends. Upgrade them to make sure. + ${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements.txt +done + +# We do not need precompiled .pyc and .pyo files. +clean_pyc ${PREFIX} diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh new file mode 100755 index 00000000..79a7be35 --- /dev/null +++ b/docker/build_scripts/install-runtime-packages.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# Install packages that will be needed at runtime + +# Stop at any error, show all commands +set -exuo pipefail + + +# Libraries that are allowed as part of the manylinux2014 profile +# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy +# On RPM-based systems, they are provided by these packages: +# Package: Libraries +# glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 +# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 +# libICE: libICE.so.6 +# libX11: libX11.so.6 +# libXext: libXext.so.6 +# libXrender: libXrender.so.1 +# libgcc: libgcc_s.so.1 +# libstdc++: libstdc++.so.6 +# mesa: libGL.so.1 +# +# PEP is missing the package for libSM.so.6 for RPM based system +# Install development packages (except for libgcc which is provided by gcc install) +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev" +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + + +# Runtime dependencies. c.f. install-build-packages.sh +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + RUNTIME_DEPS="zlib bzip2 expat ncurses readline tk gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi" + if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + RUNTIME_DEPS="${RUNTIME_DEPS} db4" + else + RUNTIME_DEPS="${RUNTIME_DEPS} libdb" + fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + RUNTIME_DEPS="zlib1g libbz2-1.0 libexpat1 libncurses5 libreadline7 tk libgdbm3 libdb5.3 libpcap0.8 liblzma5 libssl1.1 libkeyutils1 libkrb5-3 libcomerr2 libidn2-0 libcurl3 uuid libffi6" +else + echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" + exit 1 +fi + + +BASETOOLS="autoconf automake bison bzip2 diffutils file make patch unzip" +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + PACKAGE_MANAGER=yum + BASETOOLS="${BASETOOLS} which" + # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages + echo "multilib_policy=best" >> /etc/yum.conf + yum -y update + yum -y install https://archives.fedoraproject.org/pub/archive/epel/6/x86_64/epel-release-6-8.noarch.rpm curl + fixup-mirrors + TOOLCHAIN_DEPS="devtoolset-8-binutils devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-gcc-gfortran yasm" + if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then + # Software collection (for devtoolset-8) + yum -y install centos-release-scl + fixup-mirrors + elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + # Add libgfortran4 for devtoolset-7 compat + TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} libgfortran4" + # Install mayeut/devtoolset-8 repo to get devtoolset-8 + curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-8.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-8-i386/repo/custom-1/mayeut-devtoolset-8-i386-custom-1.repo + fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + PACKAGE_MANAGER=yum + BASETOOLS="${BASETOOLS} which" + # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages + echo "multilib_policy=best" >> /etc/yum.conf + # Error out if requested packages do not exist + echo "skip_missing_names_on_install=False" >> /etc/yum.conf + # Make sure that locale will not be removed + sed -i '/^override_install_langs=/d' /etc/yum.conf + if [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then + # workaround for https://github.com/nealef/clefos/issues/5 + # this shall be removed ASAP + yum -y install epel-release-7-12 + yum -y --exclude=epel-release update + else + yum -y update + fi + yum -y install yum-utils curl + yum-config-manager --enable extras + TOOLCHAIN_DEPS="devtoolset-9-binutils devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-gcc-gfortran" + if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + # EPEL support (for yasm) + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" + elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ] || [ "${AUDITWHEEL_ARCH}" == "ppc64le" ] || [ "${AUDITWHEEL_ARCH}" == "s390x" ]; then + # Software collection (for devtoolset-9) + yum -y install centos-release-scl-rh + elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + # No yasm on i686 + # Install mayeut/devtoolset-9 repo to get devtoolset-9 + curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo + fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + PACKAGE_MANAGER=apt + export DEBIAN_FRONTEND=noninteractive + sed -i 's/none/en_US/g' /etc/apt/apt.conf.d/docker-no-languages + apt-get update -qq + apt-get upgrade -qq -y + apt-get install -qq -y --no-install-recommends ca-certificates gpg curl locales + TOOLCHAIN_DEPS="binutils gcc g++ gfortran" +fi + +if ! which localedef &> /dev/null; then + # somebody messed up glibc-common package to squeeze image size, reinstall the package + if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y reinstall glibc-common + else + echo "Not implemented" + exit 1 + fi +fi + +# upgrading glibc-common can end with removal on en_US.UTF-8 locale +localedef -i en_US -f UTF-8 en_US.UTF-8 + +if [ ${PACKAGE_MANAGER} == yum ]; then + yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} + yum clean all + rm -rf /var/cache/yum +elif [ ${PACKAGE_MANAGER} == apt ]; then + apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} + apt-get clean -qq + rm -rf /var/lib/apt/lists/* +else + echo "Not implemented" + exit 1 +fi + + +# Fix libc headers to remain compatible with C99 compilers. +find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + + +if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then + # remove useless things that have been installed by devtoolset + rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man + find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi + +rm -rf /usr/share/backgrounds + +# if we updated glibc, we need to strip locales again... +if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then + localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive +fi +if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then + mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl + build-locale-archive +else + rm /usr/lib/locale/locale-archive + localedef -i en_US -f UTF-8 en_US.UTF-8 + update-locale LANG=en_US.UTF-8 +fi + +find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +if [ -d /usr/local/share/locale ]; then + find /usr/local/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi +if [ -d /usr/local/share/man ]; then + rm -rf /usr/local/share/man +fi diff --git a/docker/build_scripts/manylinux-check.py b/docker/build_scripts/manylinux-check.py index a215f11e..ad84347a 100644 --- a/docker/build_scripts/manylinux-check.py +++ b/docker/build_scripts/manylinux-check.py @@ -1,8 +1,44 @@ -# Logic copied from PEP 599 +# Logic copied from PEP 513, PEP 599 import sys +def is_manylinux1_compatible(): + # Only Linux, and only x86-64 / i686 + from distutils.util import get_platform + if get_platform() not in ["linux-x86_64", "linux-i686"]: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. + return have_compatible_glibc(2, 5) + + +def is_manylinux2010_compatible(): + # Only Linux, and only x86-64 / i686 + from distutils.util import get_platform + if get_platform() not in ["linux-x86_64", "linux-i686"]: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2010_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. + return have_compatible_glibc(2, 12) + + def is_manylinux2014_compatible(): # Only Linux, and only supported architectures from distutils.util import get_platform @@ -58,9 +94,27 @@ def have_compatible_glibc(major, minimum_minor): return True -if is_manylinux2014_compatible(): - print("%s is manylinux2014 compatible" % (sys.executable,)) - sys.exit(0) -else: - print("%s is NOT manylinux2014 compatible" % (sys.executable,)) - sys.exit(1) +exit_code = 0 + +if sys.argv[2] in {"x86_64", "i686"} and (sys.argv[1] in {"manylinux1", "manylinux2010", "manylinux2014"} or sys.argv[1].startswith("manylinux_")): + if is_manylinux1_compatible(): + print("%s %s is manylinux1 compatible" % (sys.argv[1], sys.executable)) + else: + print("%s %s is NOT manylinux1 compatible" % (sys.argv[1], sys.executable)) + exit_code = 1 + +if sys.argv[2] in {"x86_64", "i686"} and (sys.argv[1] in {"manylinux2010", "manylinux2014"} or sys.argv[1].startswith("manylinux_")): + if is_manylinux2010_compatible(): + print("%s %s is manylinux2010 compatible" % (sys.argv[1], sys.executable)) + else: + print("%s %s is NOT manylinux2010 compatible" % (sys.argv[1], sys.executable)) + exit_code = 1 + +if sys.argv[1] in {"manylinux2014"} or sys.argv[1].startswith("manylinux_"): + if is_manylinux2014_compatible(): + print("%s %s is manylinux2014 compatible" % (sys.argv[1], sys.executable)) + else: + print("%s %s is NOT manylinux2014 compatible" % (sys.argv[1], sys.executable)) + exit_code = 1 + +sys.exit(exit_code) diff --git a/docker/manylinux-entrypoint b/docker/manylinux-entrypoint new file mode 100755 index 00000000..06ea40ef --- /dev/null +++ b/docker/manylinux-entrypoint @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +if [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + linux32 "$@" +else + exec "$@" +fi diff --git a/manylinux2010_x86_64_glibc/Dockerfile b/manylinux2010_x86_64_glibc/Dockerfile new file mode 100644 index 00000000..316302a7 --- /dev/null +++ b/manylinux2010_x86_64_glibc/Dockerfile @@ -0,0 +1,41 @@ +FROM centos:6 +LABEL maintainer="The Manylinux project" + +# do not install debuginfo and what x86_64 already provides +COPY --from=quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest \ + /rpms/glibc-2.12-1.212.1.el6.i686.rpm \ + #/rpms/glibc-common-2.12-1.212.1.el6.i686.rpm \ + #/rpms/glibc-debuginfo-2.12-1.212.1.el6.i686.rpm \ + #/rpms/glibc-debuginfo-common-2.12-1.212.1.el6.i686.rpm \ + /rpms/glibc-devel-2.12-1.212.1.el6.i686.rpm \ + #/rpms/glibc-headers-2.12-1.212.1.el6.i686.rpm \ + /rpms/glibc-static-2.12-1.212.1.el6.i686.rpm \ + #/rpms/glibc-utils-2.12-1.212.1.el6.i686.rpm \ + #/rpms/nscd-2.12-1.212.1.el6.i686.rpm \ + /rpms/ +# do not install debuginfo +COPY --from=quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest \ + /rpms/glibc-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/glibc-common-2.12-1.212.1.el6.x86_64.rpm \ + #/rpms/glibc-debuginfo-2.12-1.212.1.el6.x86_64.rpm \ + #/rpms/glibc-debuginfo-common-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/glibc-devel-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/glibc-headers-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/glibc-static-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/glibc-utils-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/nscd-2.12-1.212.1.el6.x86_64.rpm \ + /rpms/ + +# Centos 6 is EOL and is no longer available from the usual mirrors, so switch +# to https://vault.centos.org +RUN sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf && \ + sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/*.repo && \ + sed -i 's;^#baseurl=http://mirror;baseurl=https://vault;g' /etc/yum.repos.d/*.repo + +RUN yum install -y glibc.i686 glibc-devel.i686 glibc-static.i686 glibc.x86_64 glibc-devel.x86_64 glibc-static.x86_64 && \ + yum -y install /rpms/* && rm -rf /rpms && yum -y clean all && rm -rf /var/cache/yum/* && \ + # if we updated glibc, we need to strip locales again... + localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive && \ + mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl && \ + build-locale-archive && \ + find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf diff --git a/manylinux2010_x86_64_glibc/Dockerfile-i686 b/manylinux2010_x86_64_glibc/Dockerfile-i686 new file mode 100644 index 00000000..25670873 --- /dev/null +++ b/manylinux2010_x86_64_glibc/Dockerfile-i686 @@ -0,0 +1,14 @@ +FROM i386/centos:6 + + +# Centos 6 is EOL and is no longer available from the usual mirrors, so switch +# to https://vault.centos.org +RUN sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf && \ + sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/*.repo && \ + sed -i 's;^#baseurl=http://mirror;baseurl=https://vault;g' /etc/yum.repos.d/*.repo + +RUN echo "i386" > /etc/yum/vars/basearch +RUN yum -y update && \ + yum install -y util-linux-ng +COPY ./build_scripts /build_scripts +RUN linux32 bash /build_scripts/rebuild-glibc-without-vsyscall.sh diff --git a/manylinux2010_x86_64_glibc/Dockerfile-x86_64 b/manylinux2010_x86_64_glibc/Dockerfile-x86_64 new file mode 100644 index 00000000..a8d014b5 --- /dev/null +++ b/manylinux2010_x86_64_glibc/Dockerfile-x86_64 @@ -0,0 +1,10 @@ +FROM centos:6 + +# Centos 6 is EOL and is no longer available from the usual mirrors, so switch +# to https://vault.centos.org +RUN sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf && \ + sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/*.repo && \ + sed -i 's;^#baseurl=http://mirror;baseurl=https://vault;g' /etc/yum.repos.d/*.repo + +COPY ./build_scripts /build_scripts +RUN bash /build_scripts/rebuild-glibc-without-vsyscall.sh diff --git a/manylinux2010_x86_64_glibc/README.rst b/manylinux2010_x86_64_glibc/README.rst new file mode 100644 index 00000000..c4b0fc37 --- /dev/null +++ b/manylinux2010_x86_64_glibc/README.rst @@ -0,0 +1,79 @@ +centos-6-no-vsyscall +==================== + +*Summary*: Because of +https://mail.python.org/pipermail/wheel-builders/2016-December/000239.html, +this a CentOS 6.10 Docker image that rebuilds ``glibc`` without +*vsyscall* is necessary to reliably run ``manylinux2010`` on 64-bit +hosts. This requires building the image on a system with +``vsyscall=emulate`` but allows the resulting container to run on +systems with ``vsyscall=none`` or ``vsyscall=emulate``. + +*vsyscall* is an antiquated optimization for a small number of +frequently-used system calls. A vsyscall-enabled Linux kernel maps a +read-only page of data and system calls into a process' memory at a +fixed address. These system calls can then be invoked by +dereferencing a function pointers to fixed offsets in that page, +saving a relatively expensive context switch. [1]_ + +Unfortunately, because the code and its location in memory are fixed +and well-known, the vsyscall mechanism has become a source of gadgets +for ROP attacks (specifically, Sigreturn-Oriented Programs). [2]_ +Linux 3.1 introduced vsyscall emulation that prevents attackers from +jumping into the middle of the system calls' code at the expense of +speed, as well as the ability to disable it entirely. [3]_ [4]_ The +vsyscall mechanism could not be eliminated at the time because +``glibc`` versions earlier than 2.14 contained hard-coded references +to the fixed memory address, specifically in ``time(2)``. [5]_ These +segfault when attempting to issue a vsyscall-optimized system call +against a kernel that has disabled it. + +Linux introduced a "virtual dynamic shared object" (vDSO) that +achieves the same high-speed, in-process system call mechanism via +shared objects sometime before the kernel's migration to git. While +old itself, vDSO 's presentation as a shared library allows it to +benefit from ASLR on modern systems, making it no more amenable to ROP +gadgets than any other shared library. ``glibc`` only switched over +completely to vDSO as of glibc 2.25, so until recently vsyscall +emulation has remained on for most kernels. [6]_ Furthermore, i686 +does not use vsyscall at all, so no version of ``glibc`` requires +patching on that architecture. + +At the same time, vsyscall emulation still exposed values useful to +ROP attacks, so Linux 4.4 added a compilation option to disable +it. [7]_ [8]_ Distributions are beginning to ship kernels configured +without vsyscall, and running CentOS 5 (``glibc`` 2.5) or 6 (``glibc`` +2.12) Docker containers on these distributions indeed causes segfaults +without ``vsyscall=emulate`` [9]_ [10]_. CentOS 6, however, is +supported until 2020. It is likely that more and more distributions +will ship with ``CONFIG_LEGACY_VSYSCALL_NONE``; if managed CI services +like Travis make this switch, developers will be unable to build +``manylinux2010`` wheels with our Docker image. + +Fortunately, vsyscall is merely an optimization, and patches that +remove it can be backported to glibc 2.12 and the library recompiled. +The result is this Docker image. It can be run on kernels regardless +of their vsyscall configuration because executable and libraries on +CentOS are dynamically linked against glibc. Libraries built on this +image are unaffected because: + +a) the kernel only maps vsyscall pages into processes; +b) only glibc used the vsyscall interface directly, and it's + included in manylinux2010's whitelist policy. + +Developers who build this vsyscall-less Docker image itself, however, +must do so on a system with ``vsyscall=emulate``. + +References: +=========== + +.. [1] https://lwn.net/Articles/446528/ +.. [2] http://www.cs.vu.nl/~herbertb/papers/srop_sp14.pdf +.. [3] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=5cec93c216db77c45f7ce970d46283bcb1933884 +.. [4] https://www.kernel.org/pub/linux/kernel/v3.x/ChangeLog-3.1 +.. [5] https://sourceware.org/git/?p=glibc.git;a=blob;f=ChangeLog;h=3a6abda7d07fdaa367c48a9274cc1c08498964dc;hb=356f8bc660a154a07b03da7c536831da5c8f74fe +.. [6] https://sourceware.org/git/?p=glibc.git;a=blob;f=ChangeLog;h=6037fef737f0338a84c6fb564b3b8dc1b1221087;hb=58557c229319a3b8d2eefdb62e7df95089eabe37 +.. [7] https://googleprojectzero.blogspot.fr/2015/08/three-bypasses-and-fix-for-one-of.html +.. [8] https://outflux.net/blog/archives/2016/09/27/security-things-in-linux-v4-4/ +.. [9] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852620#20 +.. [10] https://github.com/CentOS/sig-cloud-instance-images/issues/62 diff --git a/manylinux2010_x86_64_glibc/build.sh b/manylinux2010_x86_64_glibc/build.sh new file mode 100755 index 00000000..697c69d8 --- /dev/null +++ b/manylinux2010_x86_64_glibc/build.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -e -u -x -o pipefail + +if [ "${PLATFORM-}" != x86_64 ]; then + echo "This script is useful only on x86_64" + exit 0 +fi + +basedir=$(dirname "$0") + +docker_build() { + # Output something every 10 minutes or Travis kills the job + while sleep 9m; do echo -n -e " \b"; done & + sleep_loop_pid=$! + docker build --rm "$@" "$basedir" + kill "$sleep_loop_pid" +} + +case "${1-}" in +32) + docker_build \ + -f "$basedir/Dockerfile-i686" \ + -t quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest \ + --cache-from quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest + ;; +64) + docker_build \ + -f "$basedir/Dockerfile-x86_64" \ + -t quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest \ + --cache-from quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest + ;; +all) + docker_build \ + -t quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest \ + --cache-from quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest \ + --cache-from quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest \ + --cache-from quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest + ;; +*) + echo "Usage: $0 {32|64|all}" >&2 + exit 1 + ;; +esac diff --git a/manylinux2010_x86_64_glibc/build_scripts/CentOS-source.repo b/manylinux2010_x86_64_glibc/build_scripts/CentOS-source.repo new file mode 100644 index 00000000..9865a1a7 --- /dev/null +++ b/manylinux2010_x86_64_glibc/build_scripts/CentOS-source.repo @@ -0,0 +1,7 @@ +[base-source] +name=CentOS-6.10 - Base SRPMS +baseurl=https://vault.centos.org/6.10/os/Source/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 +priority=1 +enabled=1 diff --git a/manylinux2010_x86_64_glibc/build_scripts/glibc.spec.patch b/manylinux2010_x86_64_glibc/build_scripts/glibc.spec.patch new file mode 100644 index 00000000..c7a3b0f9 --- /dev/null +++ b/manylinux2010_x86_64_glibc/build_scripts/glibc.spec.patch @@ -0,0 +1,29 @@ +diff --git a/SPECS/glibc.spec b/SPECS/glibc.spec +index 9bd07c9..c389711 100644 +--- a/SPECS/glibc.spec ++++ b/SPECS/glibc.spec +@@ -1,6 +1,6 @@ + %define glibcsrcdir glibc-2.12-2-gc4ccff1 + %define glibcversion 2.12 +-%define glibcrelease 1.212%{?dist} ++%define glibcrelease 1.212.1%{?dist} + %define run_glibc_tests 1 + %define auxarches athlon sparcv9v sparc64v alphaev6 + %define xenarches i686 athlon +@@ -279,6 +279,7 @@ + Patch247: glibc-rh1452717-4.patch + Patch248: glibc-rh1504810-1.patch + Patch249: glibc-rh1504810-2.patch ++Patch250: remove-vsyscall.patch + + Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + Obsoletes: glibc-profile < 2.4 +@@ -731,6 +732,7 @@ + %patch247 -p1 + %patch248 -p1 + %patch249 -p1 ++%patch250 -E -p3 + + # A lot of programs still misuse memcpy when they have to use + # memmove. The memcpy implementation below is not tolerant at + diff --git a/manylinux2010_x86_64_glibc/build_scripts/rebuild-glibc-without-vsyscall.sh b/manylinux2010_x86_64_glibc/build_scripts/rebuild-glibc-without-vsyscall.sh new file mode 100644 index 00000000..9b2b570c --- /dev/null +++ b/manylinux2010_x86_64_glibc/build_scripts/rebuild-glibc-without-vsyscall.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Prep script for x86_64 that recompiles glibc without vsyscalls. + +# Stop at any error, show all commands +set -ex + +# Locate the prep directory +MY_DIR=/$(dirname "${BASH_SOURCE[0]}") + +# glibc versions +ORIGINAL_GLIBC_VERSION=2.12-1.212 +PATCHED_GLIBC_VERSION=2.12-1.212.1 + +# Source RPM topdir +SRPM_TOPDIR=/root/rpmbuild + +# Source RPM download directory +DOWNLOADED_SRPMS=/root/srpms + +# Include the CentOS source RPM repository. +# https://bugs.centos.org/view.php?id=1646 +cp $MY_DIR/CentOS-source.repo /etc/yum.repos.d/CentOS-source.repo + +# Extract and prepare the source +# https://blog.packagecloud.io/eng/2015/04/20/working-with-source-rpms/ +yum -y update +yum -y install yum-utils rpm-build +yum-builddep -y glibc +mkdir $DOWNLOADED_SRPMS +# The glibc RPM's contents are owned by mockbuild +adduser mockbuild +# yumdownloader assumes the current working directory +(cd $DOWNLOADED_SRPMS && yumdownloader --source glibc) +rpm -ivh $DOWNLOADED_SRPMS/glibc-$ORIGINAL_GLIBC_VERSION.el6.src.rpm +# Prepare the source by applying Red Hat and CentOS patches +rpmbuild -bp $SRPM_TOPDIR/SPECS/glibc.spec + +# Copy the vsyscall removal patch into place +cp $MY_DIR/remove-vsyscall.patch $SRPM_TOPDIR/SOURCES +# Patch the RPM spec file so that it uses the vsyscall removal patch +(cd $SRPM_TOPDIR/SPECS && patch -p2 < $MY_DIR/glibc.spec.patch) + +# Build the RPMS +# In case of error, you can `docker commit` to inspect the build.log +rpmbuild -ba $SRPM_TOPDIR/SPECS/glibc.spec >> /var/log/build.log + +mv $SRPM_TOPDIR/RPMS/* /rpms/ + +# Show us what happened last before cleaning up the log +echo ~~~~~~~~~~~~~~~~~~~~~ final lines of the build log ~~~~~~~~~~~~~~~~~~~~~ >/dev/null +tail -n30 /var/log/build.log +rm /var/log/build.log diff --git a/manylinux2010_x86_64_glibc/build_scripts/remove-vsyscall.patch b/manylinux2010_x86_64_glibc/build_scripts/remove-vsyscall.patch new file mode 100644 index 00000000..15f4fdcc --- /dev/null +++ b/manylinux2010_x86_64_glibc/build_scripts/remove-vsyscall.patch @@ -0,0 +1,401 @@ +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +index 22beaec..d1e29da 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +@@ -68,10 +68,6 @@ + #endif + + +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- +- + .globl __lll_lock_wait_private + .type __lll_lock_wait_private,@function + .hidden __lll_lock_wait_private +@@ -250,10 +246,9 @@ __lll_timedlock_wait: + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- /* This is a regular function call, all caller-save registers +- might be clobbered. */ +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +@@ -402,8 +397,9 @@ __lll_timedwait_tid: + /* Get current time. */ + 2: movq %rsp, %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S +index b6537f9..cf9121d 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S +@@ -51,9 +51,6 @@ + orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg + #endif + +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- + + .globl __lll_robust_lock_wait + .type __lll_robust_lock_wait,@function +@@ -220,10 +217,9 @@ __lll_robust_timedlock_wait: + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- /* This is a regular function call, all caller-save registers +- might be clobbered. */ +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +index fecaa50..9ea8353 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +@@ -26,9 +26,6 @@ + + #include + +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- + + .text + +@@ -490,13 +487,11 @@ __pthread_cond_timedwait: + movq __vdso_clock_gettime@GOTPCREL(%rip), %rax + movq (%rax), %rax + PTR_DEMANGLE (%rax) +- jz 26f + call *%rax +- jmp 27f +-# endif +-26: movl $__NR_clock_gettime, %eax ++# else ++ movl $__NR_clock_gettime, %eax + syscall +-27: ++# endif + # ifndef __ASSUME_POSIX_TIMERS + cmpq $-ENOSYS, %rax + je 19f +@@ -510,8 +505,9 @@ __pthread_cond_timedwait: + # else + leaq 24(%rsp), %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 40(%rsp), %rax +@@ -648,8 +644,9 @@ __pthread_cond_timedwait: + /* clock_gettime not available. */ + 19: leaq 32(%rsp), %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 40(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +index 22a4744..f65d976 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +@@ -23,10 +23,6 @@ + #include + #include + +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- + .text + + .globl pthread_rwlock_timedrdlock +@@ -123,8 +119,9 @@ pthread_rwlock_timedrdlock: + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +index 20a9c00..4338e02 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +@@ -23,10 +23,6 @@ + #include + #include + +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- + .text + + .globl pthread_rwlock_timedwrlock +@@ -120,8 +116,9 @@ pthread_rwlock_timedwrlock: + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S +index c339494..30e67e2 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S ++++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S +@@ -24,10 +24,6 @@ + #include + #include + +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- + .text + + .globl sem_timedwait +@@ -212,9 +208,10 @@ sem_timedwait: + addq $1, NWAITERS(%r12) + + 7: xorl %esi, %esi +- movq %rsp, %rdi +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +- callq *%rax ++ movq %rsp,%rdi ++ /* This call works because we directly jump to a system call entry ++ which preserves all the registers. */ ++ call JUMPTARGET(__gettimeofday) + + /* Compute relative timeout. */ + movq 8(%rsp), %rax +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S +deleted file mode 100644 +index 18ec6db..0000000 +--- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S ++++ /dev/null +@@ -1,50 +0,0 @@ +-/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc. +- This file is part of the GNU C Library. +- +- The GNU C Library is free software; you can redistribute it and/or +- modify it under the terms of the GNU Lesser General Public +- License as published by the Free Software Foundation; either +- version 2.1 of the License, or (at your option) any later version. +- +- The GNU C Library is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- Lesser General Public License for more details. +- +- You should have received a copy of the GNU Lesser General Public +- License along with the GNU C Library; if not, write to the Free +- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +- 02111-1307 USA. */ +- +-#include +-#define _ERRNO_H 1 +-#include +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 +- +- +-ENTRY (__gettimeofday) +- /* Align stack. */ +- sub $0x8, %rsp +- cfi_adjust_cfa_offset(8) +-#ifdef SHARED +- movq __vdso_gettimeofday(%rip), %rax +- PTR_DEMANGLE (%rax) +-#else +- movq $VSYSCALL_ADDR_vgettimeofday, %rax +-#endif +- callq *%rax +- /* Check error return. */ +- cmpl $-4095, %eax +- jae SYSCALL_ERROR_LABEL +- +-L(pseudo_end): +- add $0x8, %rsp +- cfi_adjust_cfa_offset(-8) +- ret +-PSEUDO_END(__gettimeofday) +- +-libc_hidden_def (__gettimeofday) +-weak_alias (__gettimeofday, gettimeofday) +-libc_hidden_weak (gettimeofday) +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c +index ead7dbc..08c1ef7 100644 +--- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c ++++ b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c +@@ -17,6 +17,7 @@ + 02111-1307 USA. */ + + #ifdef SHARED ++# include + # include + # include + +@@ -26,6 +27,8 @@ long int (*__vdso_clock_gettime) (clockid_t, struct timespec *) + __attribute__ ((nocommon)); + strong_alias (__vdso_clock_gettime, __GI___vdso_clock_gettime attribute_hidden) + ++extern int __gettimeofday (struct timeval *__tv, struct timezone *__tz); ++ + + static inline void + _libc_vdso_platform_setup (void) +@@ -33,10 +36,9 @@ _libc_vdso_platform_setup (void) + PREPARE_VERSION (linux26, "LINUX_2.6", 61765110); + + void *p = _dl_vdso_vsym ("gettimeofday", &linux26); +- /* If the vDSO is not available we fall back on the old vsyscall. */ +-#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000ul ++ /* If the vDSO is not available we fall back on the syscall. */ + if (p == NULL) +- p = (void *) VSYSCALL_ADDR_vgettimeofday; ++ p = __gettimeofday; + PTR_MANGLE (p); + __vdso_gettimeofday = p; + +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S +deleted file mode 100644 +index a950990..0000000 +--- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S ++++ /dev/null +@@ -1,50 +0,0 @@ +-/* Copyright (C) 2007 Free Software Foundation, Inc. +- This file is part of the GNU C Library. +- +- The GNU C Library is free software; you can redistribute it and/or +- modify it under the terms of the GNU Lesser General Public +- License as published by the Free Software Foundation; either +- version 2.1 of the License, or (at your option) any later version. +- +- The GNU C Library is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- Lesser General Public License for more details. +- +- You should have received a copy of the GNU Lesser General Public +- License along with the GNU C Library; if not, write to the Free +- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +- 02111-1307 USA. */ +- +-#include +-#include +-#define _ERRNO_H 1 +-#include +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vgetcpu 0xffffffffff600800 +- +- +-ENTRY (sched_getcpu) +- /* Align stack and create local variable for result. */ +- sub $0x8, %rsp +- cfi_adjust_cfa_offset(8) +- +- movq %rsp, %rdi +- xorl %esi, %esi +- movl $VGETCPU_CACHE_OFFSET, %edx +- addq %fs:0, %rdx +- +- movq $VSYSCALL_ADDR_vgetcpu, %rax +- callq *%rax +- +- cmpq $-4095, %rax +- jae SYSCALL_ERROR_LABEL +- +- movl (%rsp), %eax +- +-L(pseudo_end): +- add $0x8, %rsp +- cfi_adjust_cfa_offset(-8) +- ret +-PSEUDO_END(sched_getcpu) +diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S +deleted file mode 100644 +index e3f3268..0000000 +--- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S ++++ /dev/null +@@ -1,42 +0,0 @@ +-/* Copyright (C) 2001,02, 2003 Free Software Foundation, Inc. +- This file is part of the GNU C Library. +- +- The GNU C Library is free software; you can redistribute it and/or +- modify it under the terms of the GNU Lesser General Public +- License as published by the Free Software Foundation; either +- version 2.1 of the License, or (at your option) any later version. +- +- The GNU C Library is distributed in the hope that it will be useful, +- but WITHOUT ANY WARRANTY; without even the implied warranty of +- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- Lesser General Public License for more details. +- +- You should have received a copy of the GNU Lesser General Public +- License along with the GNU C Library; if not, write to the Free +- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +- 02111-1307 USA. */ +- +-#include +-#define _ERRNO_H 1 +-#include +- +-/* For the calculation see asm/vsyscall.h. */ +-#define VSYSCALL_ADDR_vtime 0xffffffffff600400 +- +- +-/* Return the current time as a `time_t' and also put it in *T if T is +- not NULL. Time is represented as seconds from Jan 1 00:00:00 1970. */ +- +-ENTRY (time) +- /* Align stack. */ +- sub $0x8, %rsp +- cfi_adjust_cfa_offset(8) +- +- movq $VSYSCALL_ADDR_vtime, %rax +- callq *%rax +- +- add $0x8, %rsp +- cfi_adjust_cfa_offset(-8) +- ret +-PSEUDO_END_NOERRNO(time) +-libc_hidden_def (time) diff --git a/travis-ci/cache_images.sh b/travis-ci/cache_images.sh new file mode 100755 index 00000000..e13bd2f9 --- /dev/null +++ b/travis-ci/cache_images.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e -u -x -o pipefail +shopt -s nullglob +# shellcheck source=tags.sh +source "$(dirname "$0")"/tags.sh +mkdir -p "$HOME"/docker +for tag in "${tags[@]}"; do + image_id=$(docker images --format '{{.ID}}' "$tag" 2>/dev/null) + if [[ -n "$image_id" ]]; then + tag_hash=$(echo -n "$tag" | md5sum | awk '{print $1}') + image_path="$HOME/docker/${tag_hash}_${image_id}.tar.gz" + if [[ ! -e "$image_path" ]]; then + for old_image_path in "$HOME/docker/${tag_hash}_"*".tar.gz"; do + rm "$old_image_path" + done + docker save "$tag" | gzip -2 >"$image_path" + fi + fi +done diff --git a/travis-ci/load_cached_images.sh b/travis-ci/load_cached_images.sh new file mode 100755 index 00000000..10a81ddc --- /dev/null +++ b/travis-ci/load_cached_images.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e -u -x -o pipefail +shopt -s nullglob +# shellcheck source=tags.sh +source "$(dirname "$0")"/tags.sh +ls -la "$HOME"/docker || true +for tag in "${tags[@]}"; do + tag_hash=$(echo -n "$tag" | md5sum | awk '{print $1}') + for image_path in "$HOME/docker/${tag_hash}_"*".tar.gz"; do + docker load --input="$image_path" || rm -f "$image_path" + done +done diff --git a/travis-ci/tags.sh b/travis-ci/tags.sh new file mode 100644 index 00000000..6369434c --- /dev/null +++ b/travis-ci/tags.sh @@ -0,0 +1,22 @@ +case "${1-}" in +32) + tags=( quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest ) + ;; +64) + tags=( quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest ) + ;; +combined) + tags=( quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest ) + ;; +all) + tags=( + quay.io/pypa/manylinux2010_centos-6-with-vsyscall32:latest + quay.io/pypa/manylinux2010_centos-6-with-vsyscall64:latest + quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest + ) + ;; +*) + echo "Usage: $0 {32|64|combined|all}" >&2 + exit 1 + ;; +esac