diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index b405d335ab5..d5935789219 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -33,15 +33,131 @@ env: jobs: - docker: + standard-pre: uses: ./.github/workflows/docker.yml with: + # Build from scratch + docker_targets: "with-system-packages configured with-targets-pre" # FIXME: duplicated from env.TARGETS targets_pre: all-sage-local + tox_packages_factors: >- + ["standard"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + + standard: + if: ${{ success() || failure() }} + needs: [standard-pre] + uses: ./.github/workflows/docker.yml + with: + # Build incrementally from previous stage (pre) + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + docker_targets: "with-targets with-targets-optional" + # FIXME: duplicated from env.TARGETS targets: build doc-html targets_optional: ptest + tox_packages_factors: >- + ["standard"] docker_push_repository: ghcr.io/${{ github.repository }}/ + minimal-pre: + if: ${{ success() || failure() }} + # It does not really "need" it. + needs: [standard] + uses: ./.github/workflows/docker.yml + with: + # Build from scratch + docker_targets: "with-system-packages configured with-targets-pre" + # FIXME: duplicated from env.TARGETS + targets_pre: all-sage-local + tox_packages_factors: >- + ["minimal"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + + minimal: + if: ${{ success() || failure() }} + needs: [minimal-pre] + uses: ./.github/workflows/docker.yml + with: + # Build incrementally from previous stage (pre) + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + docker_targets: "with-targets with-targets-optional" + # FIXME: duplicated from env.TARGETS + targets: build doc-html + targets_optional: ptest + tox_packages_factors: >- + ["minimal] + docker_push_repository: ghcr.io/${{ github.repository }}/ + + maximal-pre: + if: ${{ success() || failure() }} + needs: [minimal] + uses: ./.github/workflows/docker.yml + with: + # Build from scratch + docker_targets: "with-system-packages configured with-targets-pre" + # FIXME: duplicated from env.TARGETS + targets_pre: all-sage-local + tox_packages_factors: >- + ["maximal"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + + optional-0-o: + if: ${{ success() || failure() }} + needs: [maximal-pre] + uses: ./.github/workflows/docker.yml + with: + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + tox_packages_factors: >- + ["maximal"] + docker_targets: "with-targets-optional" + targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[0-o]))' + + + optional-p-z: + if: ${{ success() || failure() }} + needs: [optional-0-o] + uses: ./.github/workflows/docker.yml + with: + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + tox_packages_factors: >- + ["maximal"] + docker_targets: "with-targets-optional" + targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[p-z]))' + + experimental-0-o: + if: ${{ success() || failure() }} + needs: [optional-p-z] + uses: ./.github/workflows/docker.yml + with: + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + tox_packages_factors: >- + ["maximal"] + docker_targets: "with-targets-optional" + targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[0-o]))' + + experimental-p-z: + if: ${{ success() || failure() }} + needs: [experimental-0-o] + uses: ./.github/workflows/docker.yml + with: + incremental: true + from_docker_repository: ghcr.io/${{ github.repository }}/ + from_docker_target: "with-targets-pre" + tox_packages_factors: >- + ["maximal"] + docker_targets: "with-targets-optional" + targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[p-z]))' + local-ubuntu: runs-on: ubuntu-latest diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 49b97aa6e08..d2df2268199 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -76,19 +76,19 @@ jobs: # For doctesting, we use a lower parallelization to avoid timeouts. run: | case "${{ matrix.stage }}" in - 1) export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL="" + 1) export TARGETS_PRE="all-sage-local" TARGETS="all-sage-local" TARGETS_OPTIONAL="build/make/Makefile" ;; 2) export TARGETS_PRE="all-sage-local" TARGETS="build doc-html" TARGETS_OPTIONAL="ptest" ;; 2-optional*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile" targets_pattern="${{ matrix.stage }}" targets_pattern="${targets_pattern#2-optional-}" - export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :optional: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) ) + export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies | grep -v sagemath_doc | grep "^[$targets_pattern]" ) ) ;; 2-experimental*) export TARGETS_PRE="build/make/Makefile" TARGETS="build/make/Makefile" targets_pattern="${{ matrix.stage }}" targets_pattern="${targets_pattern#2-experimental-}" - export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :experimental: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep "^[$targets_pattern]" ) ) + export TARGETS_OPTIONAL=$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file 'spkg-install.in|spkg-install|requirements.txt' --no-file huge|has_nonfree_dependencies | grep -v sagemath_doc | grep "^[$targets_pattern]" ) ) ;; esac MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 23cab40539d..8eff13405f8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,24 +18,20 @@ on: default: >- ["ubuntu-trusty-toolchain-gcc_9", "ubuntu-xenial-toolchain-gcc_9", - "ubuntu-bionic", + "ubuntu-bionic-gcc_8", "ubuntu-focal", "ubuntu-jammy", "ubuntu-kinetic", - "debian-stretch", "debian-buster", "debian-bullseye", "debian-bookworm", "debian-sid", - "linuxmint-19", - "linuxmint-19.3", + "linuxmint-19-gcc_8", + "linuxmint-19.3-gcc_8", "linuxmint-20.1", "linuxmint-20.2", "linuxmint-20.3", "linuxmint-21", - "fedora-26", - "fedora-27", - "fedora-28", "fedora-29", "fedora-30", "fedora-31", @@ -51,8 +47,8 @@ on: "gentoo-python3.9", "gentoo-python3.10", "archlinux-latest", - "opensuse-15.3", - "opensuse-15.4", + "opensuse-15.3-gcc_11", + "opensuse-15.4-gcc_11", "opensuse-tumbleweed", "conda-forge", "ubuntu-bionic-i386", @@ -68,7 +64,7 @@ on: ] max_parallel: type: number - default: 20 + default: 24 # # Publishing to GitHub Packages # @@ -76,6 +72,25 @@ on: required: false type: string # + # Incremental builds + # + docker_targets: + default: "with-system-packages configured with-targets-pre with-targets with-targets-optional" + type: string + incremental: + default: false + type: boolean + from_docker_repository: + required: false + type: string + from_docker_target: + required: false + type: string + from_docker_tag: + required: false + default: "$BUILD_TAG" + type: string + # # For use in upstream CIs # upstream_artifact: @@ -104,12 +119,16 @@ jobs: tox_system_factor: ${{ fromJson(inputs.tox_system_factors) }} tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }} env: - TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} + TOX_ENV: "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}" LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - DOCKER_TARGETS: with-system-packages configured with-targets-pre with-targets with-targets-optional + DOCKER_TARGETS: ${{ inputs.docker_targets }} TARGETS_PRE: ${{ inputs.targets_pre }} TARGETS: ${{ inputs.targets }} TARGETS_OPTIONAL: ${{ inputs.targets_optional }} + FROM_DOCKER_REPOSITORY: ${{ inputs.from_docker_repository }} + FROM_DOCKER_TARGET: ${{ inputs.from_docker_target }} + FROM_DOCKER_TAG: ${{ inputs.from_docker_tag }} + steps: - name: Check out SageMath uses: actions/checkout@v2 diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml deleted file mode 100644 index 3abeec4a775..00000000000 --- a/.github/workflows/tox-experimental.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Test experimental packages with tox - -## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments, -## whenever a GitHub pull request is opened or synchronized in a repository -## where GitHub Actions are enabled. -## -## It builds and checks some sage spkgs as defined in TARGETS. -## -## A job succeeds if there is no error. -## -## The build is run with "make V=0", so the build logs of individual packages are suppressed. -## -## At the end, all package build logs that contain an error are printed out. -## -## After all jobs have finished (or are canceled) and a short delay, -## tar files of all logs are made available as "build artifacts". - -#on: [push, pull_request] - -on: - pull_request: - types: [opened, synchronize] - push: - tags: - - '*' - workflow_dispatch: - # Allow to run manually - -env: - TARGETS_PRE: build/make/Makefile - TARGETS: build/make/Makefile - # TARGETS_OPTIONAL see below - -jobs: - docker: - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 6 - matrix: - tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11] - tox_packages_factor: [maximal] - targets_pattern: [0-g, h-o, p, q-z] - env: - TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - DOCKER_TARGETS: configured with-targets with-targets-optional - # Test all non-dummy experimental packages, but do not test huge packages - # and do not test packages that require external software - TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :experimental: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep '^[${{ matrix.targets_pattern }}]' ) )" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 500 - - name: fetch tags - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt-get clean - docker rmi $(docker image ls -aq) - echo "Largest packages:" - dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50 - sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)" - df -h - - name: Install test prerequisites - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install tox - sudo apt-get clean - df -h - - name: Try to login to ghcr.io - # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable - run: | - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then - echo "DOCKER_PUSH_REPOSITORY=ghcr.io/${{ github.repository }}/" >> $GITHUB_ENV - echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV - fi - # From the docker documentation via .ci/update-env.sh: - # "A tag name must be valid ASCII and may - # contain lowercase and uppercase letters, digits, underscores, periods and - # dashes. A tag name may not start with a period or a dash and may contain a - # maximum of 128 characters." - EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` - shopt -s extglob - case "$GITHUB_REF_NAME" in - +([0-9]).+([0-9])?(.+([0-9])) ) - EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; - +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) - EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; - esac - echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV - - run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - - name: Copy logs from the docker image or build container - run: | - mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" - cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME" - if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi - if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi - if: always() - - uses: actions/upload-artifact@v1 - with: - path: artifacts - name: ${{ env.LOGS_ARTIFACT_NAME }} - if: always() - - name: Print out logs for immediate inspection - # and markup the output with GitHub Actions logging commands - run: | - .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" - if: always() - - name: List docker images - run: | - if [ -f .tox/$TOX_ENV/Dockertags ]; then - cat .tox/$TOX_ENV/Dockertags - fi - if: always() diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml deleted file mode 100644 index 72b63ef61c3..00000000000 --- a/.github/workflows/tox-optional.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Test optional packages with tox - -## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments, -## whenever a GitHub pull request is opened or synchronized in a repository -## where GitHub Actions are enabled. -## -## It builds and checks some sage spkgs as defined in TARGETS. -## -## A job succeeds if there is no error. -## -## The build is run with "make V=0", so the build logs of individual packages are suppressed. -## -## At the end, all package build logs that contain an error are printed out. -## -## After all jobs have finished (or are canceled) and a short delay, -## tar files of all logs are made available as "build artifacts". - -#on: [push, pull_request] - -on: - pull_request: - types: [opened, synchronize] - push: - tags: - - '*' - workflow_dispatch: - # Allow to run manually - -env: - TARGETS_PRE: build/make/Makefile - TARGETS: build/make/Makefile - # TARGETS_OPTIONAL see below - -jobs: - docker: - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 6 - matrix: - tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11] - tox_packages_factor: [maximal] - targets_pattern: [0-g, h-o, p, q-z] - env: - TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} - DOCKER_TARGETS: configured with-targets with-targets-optional - # Test all non-dummy optional packages, but do not test huge packages - # and do not test packages that require external software - TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (for a in spkg-install.in spkg-install requirements.txt; do sage-package list :optional: --has-file $a --no-file huge --no-file has_nonfree_dependencies; done) | grep -v ^_ | grep -v sagemath_doc | grep '^[${{ matrix.targets_pattern }}]' ) )" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 500 - - name: fetch tags - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt-get clean - docker rmi $(docker image ls -aq) - echo "Largest packages:" - dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50 - sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)" - df -h - - name: Install test prerequisites - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install tox - sudo apt-get clean - df -h - - name: Try to login to ghcr.io - # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable - run: | - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then - echo "DOCKER_PUSH_REPOSITORY=ghcr.io/${{ github.repository }}/" >> $GITHUB_ENV - echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV - fi - # From the docker documentation via .ci/update-env.sh: - # "A tag name must be valid ASCII and may - # contain lowercase and uppercase letters, digits, underscores, periods and - # dashes. A tag name may not start with a period or a dash and may contain a - # maximum of 128 characters." - EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` - shopt -s extglob - case "$GITHUB_REF_NAME" in - +([0-9]).+([0-9])?(.+([0-9])) ) - EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; - +([0-9]).+([0-9])?(.+([0-9])).@(beta|rc)+([0-9]) ) - EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; - esac - echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV - - run: | - set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - - name: Copy logs from the docker image or build container - run: | - mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" - cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME" - if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi - if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi - if: always() - - uses: actions/upload-artifact@v1 - with: - path: artifacts - name: ${{ env.LOGS_ARTIFACT_NAME }} - if: always() - - name: Print out logs for immediate inspection - # and markup the output with GitHub Actions logging commands - run: | - .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" - if: always() - - name: List docker images - run: | - if [ -f .tox/$TOX_ENV/Dockertags ]; then - cat .tox/$TOX_ENV/Dockertags - fi - if: always() diff --git a/README.md b/README.md index 605ba1c514e..7a21328c9a4 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ in the Installation Guide. 3. [Linux, WSL] Install the required minimal build prerequisites. - - Compilers: `gcc`, `gfortran`, `g++` (GCC 6.3 to 12.x and recent + - Compilers: `gcc`, `gfortran`, `g++` (GCC 8.x to 12.x and recent versions of Clang (LLVM) are supported). See the Installation Manual for a discussion of suitable compilers. diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index c64b9c76f2d..55f52a5f630 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -33,18 +33,12 @@ echo "# the :comments: separate the generated file into sections" echo "# to simplify writing scripts that customize this file" ADD="ADD $__CHOWN" RUN=RUN -case $SYSTEM in - debian*|ubuntu*) - cat < /bin/dash; # but some of the scripts in /opt/conda/etc/conda/activate.d # from conda-forge (as of 2020-01-27) contain bash-isms: @@ -166,11 +162,16 @@ EOF exit 1 ;; esac -cat <` shell, the following standard command-line development tools must be installed on your computer: -- A **C/C++ compiler**: GCC versions 6.3 to 12.x are supported. +- A **C/C++ compiler**: GCC versions 8.x to 12.x are supported. Clang (LLVM) is also supported. See also `Using alternative compilers`_. - **make**: GNU make, version 3.80 or later. Version 3.82 or later is recommended. diff --git a/tox.ini b/tox.ini index 918ce384f31..0106e693891 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,12 @@ # # This will do a complete build of the Sage distribution in a Docker container, which will take a while. # +# To do an incremental build based on the latest build of a beta version on GitHub Actions: +# +# $ tox -e docker-fedora-31-standard-incremental +# +# This will download a large (multi-gigabyte) Docker image from GitHub Packages (ghcr.io). +# # Specific 'make' targets can be given as additional arguments after "--". # For example, to only run the configuration phase: # @@ -49,7 +55,7 @@ envlist = ### - standard # Install all known system packages equivalent to standard packages that have spkg-configure.m4 ### - maximal # Install all known system packages equivalent to standard/optional packages that have spkg-configure.m4 - docker-ubuntu-trusty-minimal, + docker-ubuntu-trusty-toolchain-gcc_9-minimal, docker-debian-bullseye-standard, docker-fedora-34-standard, docker-archlinux-latest-maximal, @@ -135,6 +141,7 @@ passenv = docker: EXTRA_DOCKER_TAGS # Use DOCKER_BUILDKIT=1 for new version - for which unfortunately we cannot save failed builds as an image docker: DOCKER_BUILDKIT + docker: BUILDKIT_INLINE_CACHE # Set for example to "with-system-packages configured with-targets-pre with-targets" # to tag intermediate images. docker: DOCKER_TARGETS @@ -142,6 +149,9 @@ passenv = docker: DOCKER_PUSH_REPOSITORY # If set, we symlink this file into {envdir}/.docker/; this can be used for providing credentials for pushing docker: DOCKER_CONFIG_FILE + docker-incremental: FROM_DOCKER_REPOSITORY + docker-incremental: FROM_DOCKER_TARGET + docker-incremental: FROM_DOCKER_TAG local: MAKE local: PREFIX local: SAGE_NUM_THREADS @@ -202,7 +212,6 @@ setenv = ubuntu-bionic: IGNORE_MISSING_SYSTEM_PACKAGES=yes ubuntu-focal: BASE_TAG=focal ubuntu-jammy: BASE_TAG=jammy - ubuntu-jammy: IGNORE_MISSING_SYSTEM_PACKAGES=yes ubuntu-kinetic: BASE_TAG=kinetic ubuntu-kinetic: IGNORE_MISSING_SYSTEM_PACKAGES=yes # @@ -219,7 +228,6 @@ setenv = debian-bullseye: BASE_TAG=bullseye debian-bullseye: IGNORE_MISSING_SYSTEM_PACKAGES=yes debian-bookworm: BASE_TAG=bookworm - debian-bookworm: IGNORE_MISSING_SYSTEM_PACKAGES=yes debian-sid: BASE_TAG=sid # # https://hub.docker.com/u/linuxmintd @@ -259,9 +267,9 @@ setenv = fedora-34: BASE_TAG=34 fedora-34: IGNORE_MISSING_SYSTEM_PACKAGES=no fedora-35: BASE_TAG=35 - fedora-35: IGNORE_MISSING_SYSTEM_PACKAGES=yes + fedora-35: IGNORE_MISSING_SYSTEM_PACKAGES=no fedora-36: BASE_TAG=36 - fedora-36: IGNORE_MISSING_SYSTEM_PACKAGES=yes + fedora-36: IGNORE_MISSING_SYSTEM_PACKAGES=no fedora-37: BASE_TAG=37 fedora-37: IGNORE_MISSING_SYSTEM_PACKAGES=yes # @@ -442,7 +450,9 @@ setenv = # # Resulting full image:tag name # - docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:} + docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:} + docker-incremental: FULL_BASE_IMAGE_AND_TAG={env:FROM_DOCKER_REPOSITORY:ghcr.io/sagemath/sage/}sage-$(echo {envname} | sed 's/-incremental//')-{env:FROM_DOCKER_TARGET:with-targets}:{env:FROM_DOCKER_TAG:dev} + docker-incremental: SKIP_SYSTEM_PKG_INSTALL=yes # docker-nobootstrap: BOOTSTRAP=./bootstrap -D ### @@ -513,6 +523,8 @@ setenv = # - toolchain # gcc_spkg: CONFIG_CONFIGURE_ARGS_2=--without-system-gcc + gcc_8: CONFIG_CONFIGURE_ARGS_2=--with-system-gcc=force CC=gcc-8 CXX=g++-8 FC=gfortran-8 + gcc_8: EXTRA_SAGE_PACKAGES_2=_gcc8 gcc_9: CONFIG_CONFIGURE_ARGS_2=--with-system-gcc=force CC=gcc-9 CXX=g++-9 FC=gfortran-9 gcc_9: EXTRA_SAGE_PACKAGES_2=_gcc9 gcc_10: CONFIG_CONFIGURE_ARGS_2=--with-system-gcc=force CC=gcc-10 CXX=g++-10 FC=gfortran-10 @@ -620,9 +632,10 @@ commands = docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset docker: bash -c 'if [ x"{env:DOCKER_CONFIG_FILE:}" != x ]; then mkdir -p {envdir}/.docker && ln -sf $(realpath "{env:DOCKER_CONFIG_FILE:}") {envdir}/.docker/; fi' docker: bash -c 'for docker_target in {env:DOCKER_TARGETS:with-targets}; do \ - docker: BUILD_IMAGE={env:DOCKER_PUSH_REPOSITORY:}sage-{envname}-$docker_target; \ - docker: BUILD_TAG=$BUILD_IMAGE:$(git describe --dirty --always); \ - docker: TAG_ARGS=$(echo --tag $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo --tag $BUILD_IMAGE:$tag; done); \ + docker: BUILD_IMAGE_STEM=sage-$(echo {envname} | sed s/-incremental//); \ + docker: BUILD_IMAGE=$DOCKER_PUSH_REPOSITORY$BUILD_IMAGE_STEM-$docker_target; \ + docker: BUILD_TAG=$(git describe --dirty --always); \ + docker: TAG_ARGS=$(for tag in $BUILD_TAG {env:EXTRA_DOCKER_TAGS:}; do echo --tag $BUILD_IMAGE:$tag; done); \ docker: DOCKER_BUILDKIT={env:DOCKER_BUILDKIT:0} \ docker: docker build . -f {envdir}/Dockerfile \ docker: --target $docker_target \ @@ -636,11 +649,11 @@ commands = docker: --build-arg TARGETS_OPTIONAL="{env:TARGETS_OPTIONAL:ptest}" \ docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \ docker: if [ $status != 0 ]; then \ - docker: BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; PUSH_TAGS=$BUILD_TAG; \ + docker: BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_IMAGE:$BUILD_TAG; PUSH_TAGS=$BUILD_IMAGE:$BUILD_TAG; \ docker: else \ - docker: PUSH_TAGS=$(echo $BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo "$BUILD_IMAGE:$tag"; done); \ + docker: PUSH_TAGS=$(echo $BUILD_IMAGE:$BUILD_TAG; for tag in {env:EXTRA_DOCKER_TAGS:}; do echo "$BUILD_IMAGE:$tag"; done); \ docker: fi; \ - docker: echo $BUILD_TAG >> {envdir}/Dockertags; \ + docker: echo $BUILD_IMAGE:$BUILD_TAG >> {envdir}/Dockertags; \ docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then \ docker: echo Pushing $PUSH_TAGS; \ docker: for tag in $PUSH_TAGS; do \