Trac #29124: Add script packages build/pkgs/_prereq, build/pkgs/_boot…

…strap

As a follow-up to #29053, we move the `build/pkgs/*.txt` files to a more
systematic location.  This will simplify some scripts.

- `build/pkgs/_prereq/distros/debian.txt` etc. - records the minimal
requirements (system packages) for installing from source. (we move the
current `build/pkgs/debian.txt` etc. there)
- `build/pkgs/_bootstrap/distros/debian.txt` etc. - would record the
additional requirements for running `./bootstrap` (we move the current
`build/pkgs/debian-bootstrap.txt` there)

Related:
- #28998: Add script-type _spkg_check package that installs tox, mock,
pytest etc.
- #29557: `build/pkgs/_recommended` - records additional recommended
packages such as texlive-xetex (see
https://wiki.sagemath.org/prerequisites/Ubuntu)

Possible follow-up:
- `build/pkgs/_toolchain/dependencies` - to replace some special
Makefile targets currently in `build/make/Makefile.in`.

URL: https://trac.sagemath.org/29124
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Sébastien Labbé, Dima Pasechnik

.github/workflows/ci-cygwin-standard.yml

@@ -43,7 +43,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -113,7 +113,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -188,7 +188,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -268,7 +268,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -346,7 +346,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -424,7 +424,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -502,7 +502,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -582,7 +582,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -662,7 +662,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -740,7 +740,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -818,7 +818,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |
@@ -896,7 +896,7 @@ jobs:
       shell: bash {0}
       run: |
         choco --version
-        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/_prereq/distros/cygwin.txt ./build/pkgs/_prereq/distros/cygwin-bootstrap.txt)
         choco install $PACKAGES --source cygwin
     - name: bootstrap
       run: |

Makefile

@@ -208,7 +208,7 @@ ptestoptional: all
 ptestoptionallong: all
 	$(PTESTALL) --long --logfile=logs/ptestoptionallong.log
 
-configure: bootstrap src/doc/bootstrap configure.ac src/bin/sage-version.sh m4/*.m4 build/pkgs/*/spkg-configure.m4 build/pkgs/*/type build/pkgs/*.txt build/pkgs/*/distros/*.txt
+configure: bootstrap src/doc/bootstrap configure.ac src/bin/sage-version.sh m4/*.m4 build/pkgs/*/spkg-configure.m4 build/pkgs/*/type build/pkgs/*/distros/*.txt
 	./bootstrap -d
 
 install: all

bootstrap

@@ -85,8 +85,12 @@ bootstrap () {
                 # --enable-SPKG options for installing pip packages
                 if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
                     pkgtype="$(cat build/pkgs/$pkgname/type)"
-                    spkg_configures="$spkg_configures
-SAGE_SPKG_ENABLE([$pkgname], [$pkgtype])"
+                    # Trac #29124: Do not provide --enable-_recommended and similar
+                    case "$pkgname" in
+                        _*) ;;
+                        *)  spkg_configures="$spkg_configures
+SAGE_SPKG_ENABLE([$pkgname], [$pkgtype])" ;;
+                    esac
                 fi
     done
     for pkgname in $(./sage --package list --has-file spkg-configure.m4 | sort); do

build/bin/write-dockerfile.sh

@@ -12,9 +12,9 @@ IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
-SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} "$SAGE_ROOT"/build/pkgs/$SYSTEM{,-bootstrap}.txt))
+SYSTEM_PACKAGES=
 CONFIGURE_ARGS="--enable-option-checking "
-for PKG_BASE in $($SAGE_ROOT/sage -package list --has-file=distros/$SYSTEM.txt $SAGE_PACKAGE_LIST_ARGS); do
+for PKG_BASE in $($SAGE_ROOT/sage -package list --has-file=distros/$SYSTEM.txt $SAGE_PACKAGE_LIST_ARGS) _bootstrap; do
     PKG_SCRIPTS="$SAGE_ROOT"/build/pkgs/$PKG_BASE
     if [ -d $PKG_SCRIPTS ]; then
        SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt

build/pkgs/_bootstrap/SPKG.rst

@@ -0,0 +1,9 @@
+_bootstrap: Represents system packages required for running the top-level bootstrap script
+==========================================================================================
+
+Description
+-----------
+
+This optional script package represents the requirements (system packages)
+that are needed in addition to those represented by the ``_prereq`` package
+in order to run the top-level ``bootstrap`` script.

build/pkgs/_bootstrap/spkg-install

@@ -0,0 +1,2 @@
+#! /usr/bin/env bash
+# Nothing to do

build/pkgs/_bootstrap/type

@@ -0,0 +1 @@
+optional

build/pkgs/_prereq/SPKG.rst

@@ -0,0 +1,8 @@
+_prereq: Represents system packages required for installing SageMath from source
+================================================================================
+
+Description
+-----------
+
+This script package represents the minimal requirements (system packages)
+for installing SageMath from source.

build/pkgs/_prereq/spkg-configure.m4

@@ -0,0 +1,6 @@
+dnl We cannot check prerequisites because they are required
+dnl already for running the configure script.
+SAGE_SPKG_CONFIGURE([_prereq], [
+  dnl Just assume that they are present.
+  sage_spkg_install__prereq=no
+])

build/pkgs/_prereq/spkg-install

@@ -0,0 +1,2 @@
+#! /usr/bin/env bash
+# Nothing to do

build/pkgs/_prereq/type

@@ -0,0 +1 @@
+standard

m4/sage_spkg_collect.m4

@@ -230,8 +230,16 @@ for DIR in $SAGE_ROOT/build/pkgs/*; do
                 message="does not support check for system package; $message"
             ])
         ])
+
+    dnl Trac #29124: Do not talk about underscore club
+    case "$SPKG_NAME" in
+    _*)
+        ;;
+    *)
         formatted_message=$(printf '%-45s%s' "$SPKG_NAME-$SPKG_VERSION:" "$message")
         AC_MSG_RESULT([$formatted_message])
+        ;;
+    esac
 
         AS_VAR_POPDEF([sage_use_system])dnl
         AS_VAR_POPDEF([sage_require])dnl

src/doc/bootstrap

@@ -26,7 +26,7 @@ mkdir -p "$OUTPUT_DIR"
 shopt -s extglob
 
 for SYSTEM in arch debian fedora cygwin homebrew conda; do
-    SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/$SYSTEM.txt))
+    SYSTEM_PACKAGES=
     OPTIONAL_SYSTEM_PACKAGES=
     SAGELIB_SYSTEM_PACKAGES=
     SAGELIB_OPTIONAL_SYSTEM_PACKAGES=

src/doc/en/developer/portability_testing.rst

@@ -155,14 +155,14 @@ Using Sage's database of distribution prerequisites
 
 The source code of the Sage distribution contains a database of
 package names in various distributions' package managers.  For
-example, the file ``build/pkgs/debian.txt`` contains the following
+example, the file ``build/pkgs/_prereq/distros/debian.txt`` contains the following
 
 .. code-block:: yaml
 
-  # This file, build/pkgs/debian.txt, contains names of Debian/Ubuntu packages
-  # needed for installation of Sage from source.
+  # This file, build/pkgs/_prereq/distros/debian.txt, contains names
+  # of Debian/Ubuntu packages needed for installation of Sage from source.
   #
-  # In addition, the files build/pkgs/SPKG/debian.txt contain the names
+  # In addition, the files build/pkgs/SPKG/distros/debian.txt contain the names
   # of packages that provide the equivalent of SPKG.
   #
   # Everything on a line after a # character is ignored.
@@ -700,7 +700,7 @@ Let us try a first variant of the ``local`` technology, the tox
 environment called ``local-direct``.  Because all builds with tox
 begin by bootstrapping the source tree, you will need autotools and
 other prerequisites installed in your system.  See
-``build/pkgs/*-bootstrap.txt`` for a list of system packages that
+``build/pkgs/_bootstrap/distros/*.txt`` for a list of system packages that
 provide these prerequisites.
 
 We start by creating a fresh (distclean) git worktree.

tox.ini

@@ -348,6 +348,7 @@ setenv =
     local:            SETENV=:
     local-nobootstrap: BOOTSTRAP=:
     local-!direct:    PATH=/usr/bin:/bin:/usr/sbin:/sbin
+    local-sudo:       __SUDO=--sudo
     # brew caches downloaded files in ${HOME}/Library/Caches. We share it between different toxenvs.
     local-homebrew:   HOMEBREW={envdir}/homebrew
     local-homebrew-usrlocal:  HOMEBREW=/usr/local
@@ -409,12 +410,7 @@ commands =
     #
     # https://docs.brew.sh/Installation
     homebrew:      bash -c 'if [ ! -x {env:HOMEBREW}/bin/brew ]; then mkdir -p {env:HOMEBREW} && cd {env:HOMEBREW} && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 ; fi'
-    # homebrew-minimal: Minimal for bootstrapping and build.
-    # gettext and pkg-config are needed so that their aclocal macros are available.
-    homebrew:      bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/homebrew*.txt`; {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
-    # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile.
-    homebrew-standard:   bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/homebrew.txt`; {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
-    homebrew-maximal:    bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/homebrew.txt`; {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
+    homebrew:      bash -c 'PACKAGES=$(build/bin/sage-get-system-packages homebrew $(PATH=build/bin:$PATH build/bin/sage-package list {env:SAGE_PACKAGE_LIST_ARGS}) _bootstrap); {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
     #
     # local-conda
     #
@@ -427,18 +423,12 @@ commands =
     local-conda:   bash -c 'cat {env:CONDARC} >> {env:CONDA_PREFIX}/.condarc'
     local-conda:   bash -c 'if [ ! -x {env:CONDA_PREFIX}/bin/conda ]; then curl -L {env:CONDA_INSTALLER_URL_BASE}{env:CONDA_INSTALLER_FILE} -C - -o {env:SHARED_CACHE_DIR}/{env:CONDA_INSTALLER_FILE} && bash {env:SHARED_CACHE_DIR}/{env:CONDA_INSTALLER_FILE} -b -f -p {env:CONDA_PREFIX}; fi'
     local-conda:   bash -c '{env:SETENV} && {env:CONDA_PREFIX}/bin/conda update -n base --yes conda'
-    local-conda:   bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/conda*.txt`; {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES'
-    # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile.
-    local-conda-standard:  bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/conda.txt`; {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES'
-    local-conda-maximal:   bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/conda.txt`; {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES'
+    local-conda:   bash -c 'PACKAGES=$(build/bin/sage-get-system-packages conda $(PATH=build/bin:$PATH build/bin/sage-package list {env:SAGE_PACKAGE_LIST_ARGS}) _bootstrap); {env:SETENV} && {env:CONDA_PREFIX}/bin/conda install --yes --quiet $PACKAGES'
     #
-    # local-sudo
+    # local-sudo:  Use sudo to run the system package commands as root
     #
     local-sudo:    bash -c '$(build/bin/sage-print-system-package-command {env:SYSTEM} update) #'
-    local-sudo:    bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/{env:SYSTEM}*.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" '
-    # -standard and -maximal are for now the same. This should be fixed when refactoring write-dockerfile.
-    local-sudo-standard:  bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" '
-    local-sudo-maximal:   bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/{env:SYSTEM}.txt`; $(build/bin/sage-print-system-package-command {env:SYSTEM} --sudo --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" '
+    local-sudo:    bash -c 'PACKAGES=$(build/bin/sage-get-system-packages {env:SYSTEM} $(PATH=build/bin:$PATH build/bin/sage-package list {env:SAGE_PACKAGE_LIST_ARGS}) _bootstrap); $(build/bin/sage-print-system-package-command {env:SYSTEM} {env:__SUDO:} --yes --no-install-recommends install $PACKAGES) || [ "$IGNORE_MISSING_SYSTEM_PACKAGES" = yes ] && echo "(ignoring errors)" '
     #
     # All "local" environments
     #