single branch rewrite

.travis.yml

@@ -1,6 +1,6 @@
 language: c
 os: linux
-dist: bionic
+dist: focal
 services:
   - docker
 
@@ -11,31 +11,86 @@ branches:
     - /^pyup-/
     - /^dependabot/
 
+cache:
+  directories:
+    - $HOME/docker
+
 env:
   global:
     # QUAY_USERNAME and QUAY_PASSWORD for docker image upload
     - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw="
     - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA="
 
+# use YAML aliases and anchors to avoid duplication in stages
+# c.f. https://github.com/travis-ci/travis-ci/issues/8295#issuecomment-454457787
+_manylinux-build: &manylinux-build
+      stage: "Build manylinux images"
+      before_install: travis-ci/load_cached_images.sh combined
+      script: PLATFORM=$PLATFORM POLICY=$POLICY TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh glibc_skip
+      deploy:
+        provider: script
+        script: ./deploy.sh
+        on:
+          branch: master
+          repo: pypa/manylinux
+
 jobs:
   include:
-   - arch: amd64
-     env: PLATFORM="x86_64"
-   - arch: amd64
-     env: PLATFORM="i686"
-   - arch: arm64
-     env: PLATFORM="aarch64"
-   - arch: ppc64le
-     env: PLATFORM="ppc64le"
-   - arch: s390x
-     env: PLATFORM="s390x"
-
-script:
-  - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh
-
-deploy:
-  provider: script
-  script: docker/deploy.sh
-  on:
-    branch: manylinux2014
-    repo: pypa/manylinux
+    - stage: "Build patched i686 glibc for manylinux2010_x86_64 image"
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux2010"
+      before_install: travis-ci/load_cached_images.sh 32
+      script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh 32
+      before_cache: travis-ci/cache_images.sh 32
+    - stage: "Build patched x86_64 glibc for manylinux2010_x86_64 image"
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux2010"
+      before_install: travis-ci/load_cached_images.sh 64
+      script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh 64
+      before_cache: travis-ci/cache_images.sh 64
+    - stage: "Add patched glibc to manylinux2010_x86_64 base image"
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux2010"
+      before_install: travis-ci/load_cached_images.sh all
+      script: PLATFORM=$PLATFORM manylinux2010_x86_64_glibc/build.sh all
+      before_cache: travis-ci/cache_images.sh combined
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux2010"
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux2014"
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="x86_64" POLICY="manylinux_2_24"
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="i686" POLICY="manylinux2010"
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="i686" POLICY="manylinux2014"
+    - <<: *manylinux-build
+      arch: amd64
+      env: PLATFORM="i686" POLICY="manylinux_2_24"
+    - <<: *manylinux-build
+      arch: arm64-graviton2
+      virt: vm
+      group: edge
+      env: PLATFORM="aarch64" POLICY="manylinux2014"
+    - <<: *manylinux-build
+      arch: arm64-graviton2
+      virt: vm
+      group: edge
+      env: PLATFORM="aarch64" POLICY="manylinux_2_24"
+    - <<: *manylinux-build
+      arch: ppc64le
+      env: PLATFORM="ppc64le" POLICY="manylinux2014"
+    - <<: *manylinux-build
+      arch: ppc64le
+      env: PLATFORM="ppc64le" POLICY="manylinux_2_24"
+    - <<: *manylinux-build
+      arch: s390x
+      env: PLATFORM="s390x" POLICY="manylinux2014"
+    - <<: *manylinux-build
+      arch: s390x
+      env: PLATFORM="s390x" POLICY="manylinux_2_24"

build.sh

@@ -1,7 +1,83 @@
 #!/bin/bash
 
 # Stop at any error, show all commands
-set -ex
+set -exuo pipefail
 
 
-docker build --rm -t quay.io/pypa/manylinux2014_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/
+# rebuild glibc for manylinux2010_x86_64
+if [ "${POLICY}_${PLATFORM}" == "manylinux2010_x86_64" ] && [ "$1" != glibc_skip ] || [ "$1" == glibc_only ]; then
+	manylinux2010_x86_64_glibc/build.sh 32
+	manylinux2010_x86_64_glibc/build.sh 64
+	if [ "$1" == "glibc_only" ]; then
+		exit 0
+	fi
+	manylinux2010_x86_64_glibc/build.sh all
+fi
+
+
+# Export variable needed by 'docker build --build-arg'
+export POLICY
+export PLATFORM
+
+
+if [ "${PLATFORM}" == "x86_64" ]; then
+	MULTIARCH_PREFIX="amd64/"
+elif [ "${PLATFORM}" == "i686" ]; then
+	MULTIARCH_PREFIX="i386/"
+elif [ "${PLATFORM}" == "aarch64" ]; then
+	MULTIARCH_PREFIX="arm64v8/"
+elif [ "${PLATFORM}" == "ppc64le" ]; then
+	MULTIARCH_PREFIX="ppc64le/"
+elif [ "${PLATFORM}" == "s390x" ]; then
+	MULTIARCH_PREFIX="s390x/"
+else
+	echo "Unsupported platform: '${PLATFORM}'"
+	exit 1
+fi
+
+if [ "${POLICY}" == "manylinux2010" ]; then
+	if [ "${PLATFORM}" == "x86_64" ]; then
+		BASEIMAGE="quay.io/pypa/manylinux2010_centos-6-no-vsyscall"
+	elif [ "${PLATFORM}" == "i686" ]; then
+		BASEIMAGE="${MULTIARCH_PREFIX}centos:6"
+	else
+		echo "Policy '${POLICY}' does not support platform '${PLATFORM}'"
+		exit 1
+	fi
+	DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-8/root"
+	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
+	if [ "${PLATFORM}" == "i686" ]; then
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib"
+	else
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+	fi
+elif [ "${POLICY}" == "manylinux2014" ]; then
+	if [ "${PLATFORM}" == "s390x" ]; then
+		BASEIMAGE="s390x/clefos:7"
+	else
+		BASEIMAGE="${MULTIARCH_PREFIX}centos:7"
+	fi
+	DEVTOOLSET_ROOTPATH="/opt/rh/devtoolset-9/root"
+	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
+	if [ "${PLATFORM}" == "i686" ]; then
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib"
+	else
+		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+	fi
+elif [ "${POLICY}" == "manylinux_2_24" ]; then
+	BASEIMAGE="${MULTIARCH_PREFIX}debian:9"
+	DEVTOOLSET_ROOTPATH=
+	PREPEND_PATH=
+	LD_LIBRARY_PATH_ARG=
+fi
+export BASEIMAGE
+export DEVTOOLSET_ROOTPATH
+export PREPEND_PATH
+export LD_LIBRARY_PATH_ARG
+
+
+docker build \
+	--build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE \
+	--build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG \
+	--rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${TRAVIS_COMMIT} \
+	-f docker/Dockerfile docker/

docker/deploy.sh → deploy.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-tag="quay.io/pypa/manylinux2014_$PLATFORM"
+tag="quay.io/pypa/${POLICY}_${PLATFORM}"
 build_id=$(git show -s --format=%cd-%h --date=short $TRAVIS_COMMIT)
 
 docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io

docker/Dockerfile

@@ -0,0 +1,137 @@
+# default to latest policy, x86_64
+ARG BASEIMAGE=amd64/debian:9
+ARG POLICY=manylinux_2_24
+ARG PLATFORM=x86_64
+ARG DEVTOOLSET_ROOTPATH=
+ARG LD_LIBRARY_PATH_ARG=
+ARG PREPEND_PATH=
+
+
+FROM $BASEIMAGE AS runtime_base
+ARG POLICY
+ARG PLATFORM
+ARG DEVTOOLSET_ROOTPATH
+ARG LD_LIBRARY_PATH_ARG
+ARG PREPEND_PATH
+LABEL maintainer="The ManyLinux project"
+
+ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
+ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
+ENV PATH=${PREPEND_PATH}${PATH}
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+
+# first fixup mirrors, keep the script around
+COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
+RUN fixup-mirrors
+
+# setup entrypoint, this will wrap commands with `linux32` with i686 images
+COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh
+RUN /build_scripts/install-entrypoint.sh && rm -rf build_scripts
+COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
+ENTRYPOINT ["manylinux-entrypoint"]
+
+COPY build_scripts/install-runtime-packages.sh /build_scripts/install-runtime-packages.sh
+RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts
+
+COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh
+
+COPY build_scripts/install-autoconf.sh /build_scripts/install-autoconf.sh
+RUN manylinux-entrypoint /build_scripts/install-autoconf.sh
+
+COPY build_scripts/install-automake.sh /build_scripts/install-automake.sh
+RUN manylinux-entrypoint /build_scripts/install-automake.sh
+
+COPY build_scripts/install-libtool.sh /build_scripts/install-libtool.sh
+RUN manylinux-entrypoint /build_scripts/install-libtool.sh
+
+COPY build_scripts/install-patchelf.sh /build_scripts/install-patchelf.sh
+RUN manylinux-entrypoint /build_scripts/install-patchelf.sh
+
+COPY build_scripts/install-libxcrypt.sh /build_scripts/install-libxcrypt.sh
+RUN manylinux-entrypoint /build_scripts/install-libxcrypt.sh
+
+CMD ["/bin/bash"]
+
+
+FROM runtime_base AS build_base
+COPY build_scripts/install-build-packages.sh /build_scripts/install-build-packages.sh
+RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
+
+
+FROM build_base AS build_git
+COPY build_scripts/build-git.sh /build_scripts/build-git.sh
+RUN manylinux-entrypoint /build_scripts/build-git.sh
+
+
+FROM build_base AS build_cmake
+COPY build_scripts/build-cmake.sh /build_scripts/build-cmake.sh
+RUN manylinux-entrypoint /build_scripts/build-cmake.sh
+
+
+FROM build_base AS build_swig
+COPY build_scripts/build-swig.sh /build_scripts/build-swig.sh
+RUN manylinux-entrypoint /build_scripts/build-swig.sh
+
+
+FROM build_base AS build_cpython
+COPY build_scripts/build-sqlite3.sh /build_scripts/build-sqlite3.sh
+RUN manylinux-entrypoint /build_scripts/build-sqlite3.sh
+
+COPY build_scripts/build-openssl.sh /build_scripts/build-openssl.sh
+RUN manylinux-entrypoint /build_scripts/build-openssl.sh
+
+COPY build_scripts/build-cpython.sh /build_scripts/build-cpython.sh
+
+
+FROM build_cpython AS build_cpython35
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.5.10
+
+FROM build_cpython AS build_cpython36
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.12
+
+FROM build_cpython AS build_cpython37
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.9
+
+FROM build_cpython AS build_cpython38
+COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.6
+
+FROM build_cpython AS build_cpython39
+COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.1
+
+FROM runtime_base AS all_cpython
+COPY --from=build_cpython35 /opt/_internal /opt/_internal/
+COPY --from=build_cpython36 /opt/_internal /opt/_internal/
+COPY --from=build_cpython37 /opt/_internal /opt/_internal/
+COPY --from=build_cpython38 /opt/_internal /opt/_internal/
+COPY --from=build_cpython39 /opt/_internal /opt/_internal/
+COPY build_scripts/install-requirements.sh /build_scripts/install-requirements.sh
+COPY build_scripts/requirements.txt /build_scripts/requirements.txt
+RUN manylinux-entrypoint /build_scripts/install-requirements.sh
+
+
+FROM runtime_base
+COPY --from=build_git /manylinux/git /
+COPY --from=build_cmake /manylinux/cmake /
+COPY --from=build_swig /manylinux/swig /
+COPY --from=build_cpython /manylinux/sqlite3 /
+COPY --from=all_cpython /opt/_internal /opt/_internal/
+COPY build_scripts/finalize.sh /build_scripts/finalize.sh
+COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py
+COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py
+COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py
+COPY build_scripts/requirements.txt /build_scripts/requirements.txt
+COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt
+RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem