Skip to content
This repository has been archived by the owner on Jan 30, 2023. It is now read-only.

Commit

Permalink
Merge #34228
Browse files Browse the repository at this point in the history
  • Loading branch information
Matthias Koeppe committed Aug 4, 2022
2 parents d46f190 + 5decadd commit 49ebf1b
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 62 deletions.
111 changes: 56 additions & 55 deletions build/bin/write-dockerfile.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,12 @@ echo "# the :comments: separate the generated file into sections"
echo "# to simplify writing scripts that customize this file"
ADD="ADD $__CHOWN"
RUN=RUN
case $SYSTEM in
debian*|ubuntu*)
cat <<EOF
ARG BASE_IMAGE=ubuntu:latest
cat <<EOF
ARG BASE_IMAGE
FROM \${BASE_IMAGE} as with-system-packages
RUN (yes | unminimize) || echo "(ignored)"
EOF
if [ -n "$DIST_UPGRADE" ]; then
cat <<EOF
RUN sed -i.bak $DIST_UPGRADE /etc/apt/sources.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
EOF
fi
case $SYSTEM in
debian*|ubuntu*)
if [ -n "$__SUDO" ]; then
SUDO="sudo"
else
Expand All @@ -54,101 +48,102 @@ EOF
UPDATE="$SUDO apt-get update &&"
INSTALL="$SUDO DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
CLEAN="&& $SUDO apt-get clean"
if [ -n "$EXTRA_REPOSITORY" ]; then
cat <<EOF
RUN $UPDATE $INSTALL software-properties-common && ($INSTALL gpg gpg-agent || echo "(ignored)")
RUN $SUDO add-apt-repository $EXTRA_REPOSITORY
EOF
fi
if [ -n "$EXTRA_PATH" ]; then
RUN="RUN export PATH=$EXTRA_PATH:\$PATH && "
fi
case "$SKIP_SYSTEM_PKG_INSTALL" in
1|y*|Y*)
;;
*)
cat <<EOF
RUN (yes | unminimize) || echo "(ignored)"
EOF
if [ -n "$DIST_UPGRADE" ]; then
cat <<EOF
RUN sed -i.bak $DIST_UPGRADE /etc/apt/sources.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
EOF
fi
if [ -n "$EXTRA_REPOSITORY" ]; then
cat <<EOF
RUN $UPDATE $INSTALL software-properties-common && ($INSTALL gpg gpg-agent || echo "(ignored)")
RUN $SUDO add-apt-repository $EXTRA_REPOSITORY
EOF
fi
esac
;;
fedora*|redhat*|centos*)
cat <<EOF
ARG BASE_IMAGE=fedora:latest
FROM \${BASE_IMAGE} as with-system-packages
EOF
EXISTS="2>/dev/null >/dev/null yum install -y --downloadonly"
INSTALL="yum install -y"
if [ -n "$DEVTOOLSET" ]; then
cat <<EOF
case "$SKIP_SYSTEM_PKG_INSTALL" in
1|y*|Y*)
;;
*)
cat <<EOF
RUN $INSTALL centos-release-scl
RUN $INSTALL devtoolset-$DEVTOOLSET
EOF
esac
RUN="RUN . /opt/rh/devtoolset-$DEVTOOLSET/enable && "
fi
;;
gentoo*)
cat <<EOF
ARG BASE_IMAGE=sheerluck/sage-on-gentoo-stage4:latest
FROM \${BASE_IMAGE} as with-system-packages
EOF
EXISTS="2>/dev/null >/dev/null emerge -f"
UPDATE="" # not needed. "FROM gentoo/portage" used instead
INSTALL="emerge -DNut --with-bdeps=y --complete-graph=y"
;;
slackware*)
# https://docs.slackware.com/slackbook:package_management
cat <<EOF
ARG BASE_IMAGE=vbatts/slackware:latest
FROM \${BASE_IMAGE} as with-system-packages
EOF
# slackpkg install ignores packages that it does not know, so we do not have to filter
EXISTS="true"
UPDATE="(yes|slackpkg update) &&"
INSTALL="slackpkg install"
;;
arch*)
# https://hub.docker.com/_/archlinux/
cat <<EOF
ARG BASE_IMAGE=archlinux:latest
FROM \${BASE_IMAGE} as with-system-packages
EOF
UPDATE="pacman -Sy &&"
EXISTS="pacman -Si"
INSTALL="pacman -Su --noconfirm"
;;
nix*)
# https://hub.docker.com/r/nixos/nix
cat <<EOF
ARG BASE_IMAGE=nixos/nix:latest
FROM \${BASE_IMAGE} as with-system-packages
case "$SKIP_SYSTEM_PKG_INSTALL" in
1|y*|Y*)
;;
*)
cat <<EOF
RUN nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs
RUN nix-channel --update
ENV PACKAGES="$SYSTEM_PACKAGES"
EOF
esac
INSTALL="nix-env --install"
RUN="RUN nix-shell --packages \$PACKAGES --run "\'
ENDRUN=\'
;;
void*)
# https://hub.docker.com/r/voidlinux/masterdir-x86_64-musl
cat <<EOF
ARG BASE_IMAGE=voidlinux:masterdir-x86_64-musl
FROM \${BASE_IMAGE} as with-system-packages
EOF
UPDATE="xbps-install -Su &&"
EXISTS="xbps-query"
INSTALL="xbps-install --yes"
;;
opensuse*)
cat <<EOF
ARG BASE_IMAGE=opensuse/leap:latest
FROM \${BASE_IMAGE} as with-system-packages
EOF
UPDATE="zypper refresh &&"
INSTALL="zypper --ignore-unknown install --no-confirm --auto-agree-with-licenses --no-recommends --details"
;;
conda*)
cat <<EOF
ARG BASE_IMAGE=continuumio/miniconda3:latest
FROM \${BASE_IMAGE} as with-system-packages
case "$SKIP_SYSTEM_PKG_INSTALL" in
1|y*|Y*)
;;
*)
cat <<EOF
ARG USE_CONDARC=condarc.yml
ADD *condarc*.yml /tmp/
RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
RUN conda update -n base conda
RUN ln -sf /bin/bash /bin/sh
EOF
esac
# On this image, /bin/sh -> /bin/dash;
# but some of the scripts in /opt/conda/etc/conda/activate.d
# from conda-forge (as of 2020-01-27) contain bash-isms:
Expand All @@ -166,11 +161,16 @@ EOF
exit 1
;;
esac
cat <<EOF

case "$SKIP_SYSTEM_PKG_INSTALL" in
1|y*|Y*)
;;

*)
cat <<EOF
#:packages:
ENV PACKAGES="$SYSTEM_PACKAGES"
EOF
case "$IGNORE_MISSING_SYSTEM_PACKAGES" in
case "$IGNORE_MISSING_SYSTEM_PACKAGES" in
no)
cat <<EOF
RUN $UPDATE $INSTALL $SYSTEM_PACKAGES $CLEAN
Expand All @@ -180,13 +180,13 @@ EOF
if [ -n "$EXISTS" ]; then
# Filter by existing packages, try to install these in one shot; fall back to one by one.
cat <<EOF
RUN $UPDATE EXISTING_PACKAGES=""; for pkg in \$PACKAGES; do echo -n .; if $EXISTS \$pkg; then EXISTING_PACKAGES="\$EXISTING_PACKAGES \$pkg"; echo -n "\$pkg"; fi; done; $INSTALL \$EXISTING_PACKAGES || (echo "Trying again one by one:"; for pkg in \$EXISTING_PACKAGES; do echo "Trying to install \$pkg"; $INSTALL \$pkg || echo "(ignoring error)"; done); : $CLEAN
RUN $UPDATE EXISTING_PACKAGES=""; for pkg in $SYSTEM_PACKAGES; do echo -n .; if $EXISTS \$pkg; then EXISTING_PACKAGES="\$EXISTING_PACKAGES \$pkg"; echo -n "\$pkg"; fi; done; $INSTALL \$EXISTING_PACKAGES || (echo "Trying again one by one:"; for pkg in \$EXISTING_PACKAGES; do echo "Trying to install \$pkg"; $INSTALL \$pkg || echo "(ignoring error)"; done); : $CLEAN
EOF
else
# Try in one shot, fall back to one by one. Separate "RUN" commands
# for caching by docker.
cat <<EOF
RUN $UPDATE $INSTALL \${PACKAGES} || echo "(ignoring error)"
RUN $UPDATE $INSTALL $SYSTEM_PACKAGES || echo "(ignoring error)"
EOF
for pkg in $SYSTEM_PACKAGES; do
cat <<EOF
Expand All @@ -203,13 +203,14 @@ EOF
*)
echo "Argument IGNORE_MISSING_SYSTEM_PACKAGES must be yes or no"
;;
esac
esac
cat <<EOF
FROM with-system-packages as bootstrapped
#:bootstrapping:
RUN mkdir -p sage
WORKDIR sage
RUN if [ -d /sage ]; then echo "### Incremental build from \$(cat /sage/VERSION.txt)" && mv /sage /sage-old && mkdir /sage && for a in local logs; do if [ -d /sage-old/\$a ]; then mv /sage-old/\$a /sage; fi; done; rm -rf /sage-old; else mkdir -p /sage; fi
WORKDIR /sage
$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
$ADD config/config.rpath config/config.rpath
$ADD src/doc/bootstrap src/doc/bootstrap
Expand All @@ -223,7 +224,7 @@ $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
FROM bootstrapped as configured
#:configuring:
RUN mkdir -p logs/pkgs; ln -s logs/pkgs/config.log config.log
RUN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
ARG EXTRA_CONFIGURE_ARGS=""
EOF
if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
Expand Down
26 changes: 19 additions & 7 deletions tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
#
# This will do a complete build of the Sage distribution in a Docker container, which will take a while.
#
# To do an incremental build based on the latest build of a beta version on GitHub Actions:
#
# $ tox -e docker-fedora-31-standard-incremental
#
# This will download a large (multi-gigabyte) Docker image from GitHub Packages (ghcr.io).
#
# Specific 'make' targets can be given as additional arguments after "--".
# For example, to only run the configuration phase:
#
Expand Down Expand Up @@ -142,6 +148,9 @@ passenv =
docker: DOCKER_PUSH_REPOSITORY
# If set, we symlink this file into {envdir}/.docker/; this can be used for providing credentials for pushing
docker: DOCKER_CONFIG_FILE
docker-incremental: FROM_DOCKER_REPOSITORY
docker-incremental: FROM_DOCKER_TARGET
docker-incremental: FROM_DOCKER_TAG
local: MAKE
local: PREFIX
local: SAGE_NUM_THREADS
Expand Down Expand Up @@ -442,7 +451,9 @@ setenv =
#
# Resulting full image:tag name
#
docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:}
docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:}
docker-incremental: FULL_BASE_IMAGE_AND_TAG={env:FROM_DOCKER_REPOSITORY:ghcr.io/sagemath/sage/}sage-$(echo {envname} | sed 's/-incremental//')-{env:FROM_DOCKER_TARGET:with-targets}:{env:FROM_DOCKER_TAG:dev}
docker-incremental: SKIP_SYSTEM_PKG_INSTALL=yes
#
docker-nobootstrap: BOOTSTRAP=./bootstrap -D
###
Expand Down Expand Up @@ -620,9 +631,10 @@ commands =
docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset
docker: bash -c 'if [ x"{env:DOCKER_CONFIG_FILE:}" != x ]; then mkdir -p {envdir}/.docker && ln -sf $(realpath "{env:DOCKER_CONFIG_FILE:}") {envdir}/.docker/; fi'
docker: bash -c 'for docker_target in {env:DOCKER_TARGETS:with-targets}; do \
docker: BUILD_IMAGE={env:DOCKER_PUSH_REPOSITORY:}sage-{envname}-$docker_target; \
docker: BUILD_TAG=$BUILD_IMAGE:$(git describe --dirty --always); \
docker: TAG_ARGS=$(echo --tag $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo --tag $BUILD_IMAGE:$tag; done); \
docker: BUILD_IMAGE_STEM=sage-$(echo {envname} | sed s/-incremental//); \
docker: BUILD_IMAGE=$DOCKER_PUSH_REPOSITORY$BUILD_IMAGE_STEM-$docker_target; \
docker: BUILD_TAG=$(git describe --dirty --always); \
docker: TAG_ARGS=$(for tag in $BUILD_TAG {env:EXTRA_DOCKER_TAGS:}; do echo --tag $BUILD_IMAGE:$tag; done); \
docker: DOCKER_BUILDKIT={env:DOCKER_BUILDKIT:0} \
docker: docker build . -f {envdir}/Dockerfile \
docker: --target $docker_target \
Expand All @@ -636,11 +648,11 @@ commands =
docker: --build-arg TARGETS_OPTIONAL="{env:TARGETS_OPTIONAL:ptest}" \
docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \
docker: if [ $status != 0 ]; then \
docker: BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; PUSH_TAGS=$BUILD_TAG; \
docker: BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_IMAGE:$BUILD_TAG; PUSH_TAGS=$BUILD_IMAGE:$BUILD_TAG; \
docker: else \
docker: PUSH_TAGS=$(echo $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo "$BUILD_IMAGE:$tag"; done); \
docker: PUSH_TAGS=$(echo $BUILD_IMAGE:$BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo "$BUILD_IMAGE:$tag"; done); \
docker: fi; \
docker: echo $BUILD_TAG >> {envdir}/Dockertags; \
docker: echo $BUILD_IMAGE:$BUILD_TAG >> {envdir}/Dockertags; \
docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then \
docker: echo Pushing $PUSH_TAGS; \
docker: for tag in $PUSH_TAGS; do \
Expand Down

0 comments on commit 49ebf1b

Please sign in to comment.