Merge #33103

.github/workflows/tox-experimental.yml

@@ -85,6 +85,20 @@ jobs:
               echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
               echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
             fi
+            # From the docker documentation via .ci/update-env.sh:
+            # "A tag name must be valid ASCII and may
+            # contain lowercase and uppercase letters, digits, underscores, periods and
+            # dashes. A tag name may not start with a period or a dash and may contain a
+            # maximum of 128 characters."
+            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
+            shopt -s extglob
+            case "$GITHUB_REF_NAME" in
+              +([0-9]).+([0-9])?(.+([0-9])) )
+                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+              +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
+                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
+            esac
+            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
       - name: Copy logs from the docker image or build container

.github/workflows/tox-optional.yml

@@ -85,6 +85,20 @@ jobs:
               echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
               echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
             fi
+            # From the docker documentation via .ci/update-env.sh:
+            # "A tag name must be valid ASCII and may
+            # contain lowercase and uppercase letters, digits, underscores, periods and
+            # dashes. A tag name may not start with a period or a dash and may contain a
+            # maximum of 128 characters."
+            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
+            shopt -s extglob
+            case "$GITHUB_REF_NAME" in
+              +([0-9]).+([0-9])?(.+([0-9])) )
+                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+              +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
+                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
+            esac
+            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
       - name: Copy logs from the docker image or build container

.github/workflows/tox.yml

@@ -38,12 +38,12 @@ jobs:
       fail-fast: false
       max-parallel: 20
       matrix:
-        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.2.1, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
+        tox_system_factor: [gitpod, ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy,  debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, centos-7, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.2.1, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
         tox_packages_factor: [minimal, standard]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
-      DOCKER_TARGETS: configured with-targets with-targets-optional
+      DOCKER_TARGETS: with-system-packages configured with-targets-pre with-targets with-targets-optional
     steps:
       - uses: actions/checkout@v2
         with:
@@ -78,6 +78,20 @@ jobs:
               echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
               echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
             fi
+            # From the docker documentation via .ci/update-env.sh:
+            # "A tag name must be valid ASCII and may
+            # contain lowercase and uppercase letters, digits, underscores, periods and
+            # dashes. A tag name may not start with a period or a dash and may contain a
+            # maximum of 128 characters."
+            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
+            shopt -s extglob
+            case "$GITHUB_REF_NAME" in
+              +([0-9]).+([0-9])?(.+([0-9])) )
+                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
+              +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) )
+                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
+            esac
+            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
       - name: Copy logs from the docker image or build container

.gitpod.yml

@@ -0,0 +1,117 @@
+# Use custom docker image. https://www.gitpod.io/docs/config-docker
+image:
+# Each of these two options works:
+# - Directly use the "-with-targets" image built by .github/workflows/tox.yml
+#   image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev
+# - or go through a custom Dockerfile, which builds a smaller image
+#   based on the "-with-system-packages" image built by .github/workflows/tox.yml
+#   with the built SAGE_LOCAL from the "-with-targets" image copied in.
+    file: docker/.gitpod.Dockerfile
+
+# We use the following layout:
+#
+# $HOME/sage            Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod).
+#                       - We delete it in every invocation of the 'before' script
+#                         and replace it by a symlink to /workspace/...
+#                         (This symlink is needed because the package-removal scripts
+#                         ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm)
+#                         hardcode SAGE_ROOT and SAGE_SRC from package installation time)
+# $HOME/sage/logs       Logs of the Docker build.
+#                       - In the first invocation of the 'before' script, we move it
+#                         to /workspace/.../logs
+# $HOME/sage-local      The configured prefix (SAGE_LOCAL) of the Sage installation.
+#                       - During the Docker build, this is the physical location.
+#                       - In the first invocation of the 'before' script, we move it
+#                         to the new physical location /workspace/.../local
+#                         (because gitpod only preserves the contents of /workspace)
+#                         and replace it by a symlink to the new physical location.
+#                       - In subsequent invocations of the 'before' script, we
+#                         remove it and replace it by a symlink to the physical
+#                         location /workspace/.../local
+# /worktree/.../local   The physical location of the Sage installation,
+#                       established in the first run of the 'before' script and
+#                       preserved by gitpod.
+
+# Start up tasks. https://www.gitpod.io/docs/config-start-tasks/
+tasks:
+  - name: Setup
+    before: |
+      # Setup ssh key for authentication with trac
+      ## In order to use this, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$(<tempkey)"` (or by following https://www.gitpod.io/docs/environment-variables#using-the-account-settings)
+      ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac.
+      mkdir -p ~/.ssh
+      echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa
+      sed -i '/^$/d' ~/.ssh/id_rsa
+      chmod 600 ~/.ssh/id_rsa
+      unset PRIVATE_SSH_KEY
+      ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts
+      ## No need for pyenv
+      pyenv shell --unset
+      pyenv global system
+      if [ -d local ]; then
+        mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log
+        # The init script has already populated the SAGE_LOCAL,
+        # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected
+        # copy of sage-local.  Replace it again by a symlink.
+        rm -Rf $HOME/sage-local
+        ln -sf $(pwd)/local $HOME/sage-local
+        # Now run make. No timeout here.
+        MAKE='make -j24' make build V=0
+      else
+        # Prebuild init script has not been run
+        # Only /workspace is preserved during build.
+        # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace.
+        if [ -d $HOME/sage-local ]; then
+          mv $HOME/sage-local local
+        fi
+        rm -Rf $HOME/sage-local
+        ln -sf $(pwd)/local $HOME/sage-local
+        # Save the logs of the source tree used by the Docker build
+        if [ -d $HOME/sage/logs ]; then
+          mv $HOME/sage/logs logs
+        fi
+      fi
+      # Remove the source tree used by the Docker build and replace it by a symlink
+      rm -Rf $HOME/sage
+      ln -s $(pwd) $HOME/sage
+    init: |
+      # Setup trac repo
+      git remote add trac git@trac.sagemath.org:sage.git -t master
+      git remote set-url --push trac git@trac.sagemath.org:sage.git
+
+      # Start build
+      mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log
+      ./bootstrap
+      ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv
+      ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful
+      MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)"
+    env:
+      SAGE_NUM_THREADS: 8
+
+# Preinstalled VS Code extensions. https://www.gitpod.io/docs/vscode-extensions
+vscode:
+  extensions:
+    - ms-pyright.pyright
+    - ms-python.python
+    - trond-snekvik.simple-rst
+    - lextudio.restructuredtext
+    - streetsidesoftware.code-spell-checker
+    - ms-toolsai.jupyter
+
+# https://www.gitpod.io/docs/prebuilds#github-specific-configuration
+github:
+  prebuilds:
+    # enable for the default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: true
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a check to pull requests (defaults to true)
+    addCheck: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
+    addComment: true
+    # add a "Review in Gitpod" button to the pull request's description (defaults to false)
+    addBadge: true

.vscode/settings.json

@@ -1,7 +1,6 @@
 {
-    "python.venvPath": "./venv/bin/python",
     "python.defaultInterpreterPath": "./venv/bin/python",
-    "python.dataScience.jupyterServerURI": "local",
+    "python.pythonPath": "./venv/bin/python",
     "files.exclude": {
         "**/__pycache__": true,
         "src/**/*.cpp": true,

build/bin/sage-spkg

@@ -628,7 +628,7 @@ export SAGE_DESTDIR_LOCAL="${SAGE_DESTDIR}${SAGE_INST_LOCAL}"
 
 # First uninstall the previous version of this package, if any
 if [ "$KEEP_EXISTING" != "yes" ]; then
-    sage-spkg-uninstall "$PKG_BASE"
+    sage-spkg-uninstall "$PKG_BASE" "$SAGE_INST_LOCAL"
 fi
 
 # To work around #26996: Create lib and set a symlink so that writes into lib64/ end up in lib/

build/bin/write-dockerfile.sh

@@ -31,6 +31,7 @@ done
 echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
 echo "# the :comments: separate the generated file into sections"
 echo "# to simplify writing scripts that customize this file"
+ADD="ADD $__CHOWN"
 RUN=RUN
 case $SYSTEM in
     debian*|ubuntu*)
@@ -43,10 +44,15 @@ EOF
 RUN sed -i.bak $DIST_UPGRADE /etc/apt/sources.list && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
 EOF
         fi
+        if [ -n "$__SUDO" ]; then
+            SUDO="sudo"
+        else
+            unset SUDO
+        fi
         EXISTS="2>/dev/null >/dev/null apt-cache show"
-        UPDATE="apt-get update &&"
-        INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
-        CLEAN="&& apt-get clean"
+        UPDATE="$SUDO apt-get update &&"
+        INSTALL="$SUDO DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
+        CLEAN="&& $SUDO apt-get clean"
         ;;
     fedora*|redhat*|centos*)
         cat <<EOF
@@ -185,15 +191,15 @@ cat <<EOF
 
 FROM with-system-packages as bootstrapped
 #:bootstrapping:
-RUN mkdir -p /sage
-WORKDIR /sage
-ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
-ADD src/doc/bootstrap src/doc/bootstrap
-ADD src/bin src/bin
-ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/
-ADD m4 ./m4
-ADD pkgs pkgs
-ADD build ./build
+RUN mkdir -p sage
+WORKDIR sage
+$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
+$ADD src/doc/bootstrap src/doc/bootstrap
+$ADD src/bin src/bin
+$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/
+$ADD m4 ./m4
+$ADD pkgs pkgs
+$ADD build ./build
 ARG BOOTSTRAP=./bootstrap
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
 
@@ -239,7 +245,7 @@ ENV MAKE="make -j\${NUMPROC}"
 ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!gfan,!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
-ADD src src
+$ADD src src
 ARG TARGETS="build"
 $RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN
 

build/pkgs/sagelib/spkg-install

@@ -55,6 +55,10 @@ else
     time python3 -u setup.py --no-user-cfg build install || exit 1
 fi
 
+# Trac #33103: The temp.* directories are large after a full build.
+# We remove them to save space; they are not needed for fast rebuilds.
+rm -rf build/temp.*
+
 if [ "$UNAME" = "CYGWIN" ]; then
     sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null;
 fi

build/sage_bootstrap/cmdline.py

@@ -207,6 +207,9 @@ def make_parser():
         '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
         help=('only include packages that have this file in their metadata directory'
               '(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt)'))
+    parser_list.add_argument(
+        '--exclude', action='append', default=[], metavar='PACKAGE_NAME',
+        help='exclude package from list')
     parser_name = subparsers.add_parser(
         'name', epilog=epilog_name,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -321,7 +324,7 @@ def run():
     if args.subcommand == 'config':
         app.config()
     elif args.subcommand == 'list':
-        app.list_cls(*args.package_class, has_files=args.has_files)
+        app.list_cls(*args.package_class, has_files=args.has_files, exclude=args.exclude)
     elif args.subcommand == 'name':
         app.name(args.tarball_filename)
     elif args.subcommand == 'tarball':

build/sage_bootstrap/expand_class.py

@@ -24,10 +24,13 @@ class PackageClass(object):
     def __init__(self, *package_names_or_classes, **filters):
         self.names = []
         filenames = filters.pop('has_files', [])
+        excluded = filters.pop('exclude', [])
         if filters:
             raise ValueError('filter not supported')
 
         def included_in_filter(pkg):
+            if pkg.name in excluded:
+                return False
             return all(pkg.has_file(filename) for filename in filenames)
         for package_name_or_class in package_names_or_classes:
             if package_name_or_class == ':all:':

docker/.gitpod.Dockerfile

@@ -0,0 +1,12 @@
+ARG BASE_GITHUB_REPOSITORY=sagemath/sage
+ARG BASE_TAG=dev
+FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-targets:${BASE_TAG} as with-targets
+RUN sudo rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
+# Fast doc rebuilds do not work because
+# "loading pickled environment... failed; source directory has changed"
+# Until this is fixed, we can as well remove the whole documentation, which saves a lot of space.
+RUN rm -Rf /home/gitpod/sage-local/share/doc/sage
+
+FROM ghcr.io/${BASE_GITHUB_REPOSITORY}/sage-docker-gitpod-standard-with-system-packages:${BASE_TAG}
+COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage/logs  /home/gitpod/sage/logs
+COPY --chown=gitpod:gitpod --from=with-targets /home/gitpod/sage-local /home/gitpod/sage-local

src/doc/en/developer/index.rst

@@ -41,6 +41,14 @@ development!
   <http://doc.sagemath.org/html/en/installation/source.html#prerequisites>`_ on your
   system.
 
+  As an easy way to get started, you can run and edit Sage's code and contribute
+  your changes using `Gitpod <https://www.gitpod.io>`_,
+  a free online development environment based on VS Code.
+  It will launch a pre-made workspace with all dependencies and tools installed 
+  so that you can start contributing straight away.
+  Start by `going to Gitpod <https://gitpod.io/#https://github.com/sagemath/sage>`_,
+  and read :ref:`our Gitpod guidelines <section-gitpod>` to learn more.
+
 - **Conventions:** read our :ref:`conventions and guidelines
   <section-writing-code-for-sage>` for code and documentation.
 
@@ -115,6 +123,7 @@ Writing Code for Sage
 .. toctree::
    :maxdepth: 3
 
+   workspace
    coding_basics
    reviewer_checklist