Merge branch '36676+37902' into 36964

saraedum · May 23, 2024 · a436233 · a436233
2 parents a05e957 + 112a6ca
commit a436233
Show file tree

Hide file tree

Showing 582 changed files with 12,725 additions and 11,388 deletions.
diff --git a/.ci/create-changes-html.sh b/.ci/create-changes-html.sh
@@ -51,7 +51,7 @@ diffParagraphs.forEach(paragraph => {
 EOF
 echo '</head>' >> CHANGES.html
 echo '<body>' >> CHANGES.html
-(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt
+(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt
 python3 - << EOF
 import os, re, html
 with open('diff.txt', 'r') as f:

diff --git a/.ci/docker-exec-script.sh b/.ci/docker-exec-script.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+if [ $# -lt 3 ]; then
+    echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
+    exit 1
+fi
+CONTAINER=$1
+WORKDIR=$2
+shift 2
+(echo "cd \"$WORKDIR\"";
+ while [ $# -gt 1 ]; do
+     echo "export \"$1\""
+     shift
+ done;
+ cat "$1") | docker exec -i $CONTAINER bash -ex
diff --git a/.ci/merge-fixes.sh b/.ci/merge-fixes.sh
@@ -1,5 +1,8 @@
 #!/bin/sh
-# Apply open PRs labeled "blocker" from sagemath/sage as patches.
+# Apply open PRs labeled "p: CI Fix" from sagemath/sage as patches.
+# (policy set by vote in 2024-03,
+#  https://groups.google.com/g/sage-devel/c/OKwwUGyKveo/m/vpyCXYBqAAAJ)
+#
 # This script is invoked by various workflows in .github/workflows
 #
 # The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
@@ -20,15 +23,15 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
             echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
             ;;
         */*)
-            echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
+            echo "Getting open PRs with 'p: CI Fix' label from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+CI+Fix%22"
             GH="gh -R $REPO"
             REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
-            PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
+            PRs="$($GH pr list --label "p: CI Fix" --json number --jq '.[].number' | tee $REPO_FILE)"
             date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date  # Record the date, for future reference
             if [ -z "$PRs" ]; then
-                echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
+                echo "Nothing to do: Found no open PRs with 'p: CI Fix' label in $REPO."
             else
-                echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
+                echo "Found open PRs with 'p: CI Fix' label in $REPO: $(echo $PRs)"
                 git tag -f test_base
                 git commit -q -m "Uncommitted changes" --no-allow-empty -a
                 for a in $PRs; do

diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh
@@ -12,11 +12,13 @@ export GIT_AUTHOR_EMAIL="ci-sage@example.com"
 export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
 export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
 
+set -e
+
 # Set globally for other parts of the workflow
 git config --global user.name "$GIT_AUTHOR_NAME"
 git config --global user.email "$GIT_AUTHOR_EMAIL"
 
-set -ex
+set -x
 
 # If actions/checkout downloaded our source tree using the GitHub REST API
 # instead of with git (because do not have git installed in our image),
@@ -36,6 +38,10 @@ git tag -f new
 # The changed files now show up as uncommitted changes.
 # The final "git add -N" makes sure that files that were added in "new" do not show
 # as untracked files, which would be removed by "git clean -fx".
+if [ -L $WORKTREE_NAME ]; then
+    rm -f $WORKTREE_NAME
+    git worktree prune --verbose
+fi
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME

diff --git a/build/bin/write-dockerfile.sh → .ci/write-dockerfile.sh b/build/bin/write-dockerfile.sh → .ci/write-dockerfile.sh
@@ -1,20 +1,41 @@
 #! /usr/bin/env bash
-## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
-## and satisfy the requirements tested by spkg-configure.m4
-## This is called by $SAGE_ROOT/tox.ini
 set -e
-SYSTEM="${1:-debian}"
 shopt -s extglob
+##
+## Write a Dockerfile for portability testing to stdout.
+##
+## This script needs to be run from SAGE_ROOT (root of the Sage repository).
+## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...'
+##
+## Positional arguments:
+##
+SYSTEM="${1:-debian}"
 SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
-#
+##
+## Environment variables that take influence:
+##
+## - BOOTSTRAP
+## - CONFIGURE_ARGS
+## - DEVTOOLSET
+## - DIST_UPGRADE
+## - DOCKER_BUILDKIT
+## - EXTRA_PATH
+## - EXTRA_REPOSITORIES
+## - EXTRA_SYSTEM_PACKAGES
+## - FULL_BASE_IMAGE_AND_TAG
+## - SKIP_SYSTEM_PKG_INSTALL
+## - USE_CONDARC
+## - __CHOWN
+## - __SUDO
+##
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
 SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES
-CONFIGURE_ARGS="--enable-option-checking "
+SYSTEM_CONFIGURE_ARGS="--enable-option-checking "
 for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
     SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG)
     if [ -n "${SYSTEM_PACKAGE}" ]; then
@@ -24,16 +45,16 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_
             # shell-quote package if necessary
             SYSTEM_PACKAGES+=$(printf " %q" "$a")
         done
-        CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
+        SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
-echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
+echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh"
 echo "# the :comments: separate the generated file into sections"
 echo "# to simplify writing scripts that customize this file"
 ADD="ADD $__CHOWN"
 RUN=RUN
 cat <<EOF
-ARG BASE_IMAGE
+ARG BASE_IMAGE=$(eval echo "${FULL_BASE_IMAGE_AND_TAG}")
 FROM \${BASE_IMAGE} as with-system-packages
 EOF
 case $SYSTEM in
@@ -54,8 +75,21 @@ case $SYSTEM in
             1|y*|Y*)
                 ;;
             *)
+                #
+                # The Ubuntu Docker images are "minimized", meaning that some large
+                # bits such as documentation has been removed. We have to unminimize
+                # once (which reinstalls the full versions of some minimized packages),
+                # or e.g. the maxima documentation (which we depend on for correct operation)
+                # will be missing.
+                #
+                # But we only have to do this once. To save time in incremental builds,
+                # we remove the unminimize binary here after it has done its job.
+                #
                 cat <<EOF
-RUN (yes | unminimize) || echo "(ignored)"
+RUN if command -v unminimize > /dev/null; then  \
+        (yes | unminimize) || echo "(ignored)"; \
+        rm -f "$(command -v unminimize)";       \
+    fi
 EOF
                 if [ -n "$DIST_UPGRADE" ]; then
                     cat <<EOF
@@ -143,7 +177,7 @@ EOF
                 ;;
             *)
                 cat <<EOF
-ARG USE_CONDARC=condarc.yml
+ARG USE_CONDARC=${USE_CONDARC-condarc.yml}
 ADD *condarc*.yml /tmp/
 RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
 RUN conda update -n base conda
@@ -230,32 +264,47 @@ cat <<EOF
 
 FROM with-system-packages as bootstrapped
 #:bootstrapping:
-RUN if [ -d /sage ]; then echo "### Incremental build from \$(cat /sage/VERSION.txt)" && mv /sage /sage-old && mkdir /sage && for a in local logs; do if [ -d /sage-old/\$a ]; then mv /sage-old/\$a /sage; fi; done; rm -rf /sage-old; else mkdir -p /sage; fi
+$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 .gitignore /new/
+$ADD config/config.rpath /new/config/config.rpath
+$ADD src/doc/bootstrap /new/src/doc/bootstrap
+$ADD src/bin /new/src/bin
+$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/VERSION.txt /new/src/
+$ADD m4 /new/m4
+$ADD pkgs /new/pkgs
+$ADD build /new/build
+$ADD .ci /new/.ci
+$ADD .upstream.d /new/.upstream.d
+RUN if [ -d /sage ]; then                                               \
+        echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
+        if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
+            for a in local logs; do                                     \
+                if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
+            done;                                                       \
+            rm -rf /sage;                                               \
+            mv /new /sage;                                              \
+        fi;                                                             \
+    else                                                                \
+        mv /new /sage;                                                  \
+    fi
 WORKDIR /sage
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
-$ADD config/config.rpath config/config.rpath
-$ADD src/doc/bootstrap src/doc/bootstrap
-$ADD src/bin src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/VERSION.txt src/
-$ADD m4 ./m4
-$ADD pkgs pkgs
-$ADD build ./build
-$ADD .upstream.d ./.upstream.d
-ARG BOOTSTRAP=./bootstrap
+
+ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
 
 FROM bootstrapped as configured
 #:configuring:
 RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
-ARG EXTRA_CONFIGURE_ARGS=""
+ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}"
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; echo "::group::config.log"; cat config.log; echo "::endgroup::"; ./configure --enable-build-as-root; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; echo "********** configuring without forcing ***********"; ./configure \${CONFIGURE_ARGS}; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 else
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 fi
 cat <<EOF
@@ -286,7 +335,15 @@ ENV MAKE="make -j\${NUMPROC}"
 ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
-$ADD src src
+$ADD .gitignore /new/.gitignore
+$ADD src /new/src
+RUN cd /new && rm -rf .git && \
+    if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        rm -rf /sage/src;                                    \
+        mv src /sage/src;                                    \
+        cd /sage && ./bootstrap && ./config.status;          \
+    fi
+
 ARG TARGETS="build"
 $RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN $THEN_SAVE_STATUS