Merge branch '5.0.x' into strict_rpath_sanity_check

boegel · Oct 7, 2024 · 1b94db0 · 1b94db0
2 parents 1e2916d + 96ff24b
commit 1b94db0
Show file tree

Hide file tree

Showing 328 changed files with 6,909 additions and 2,950 deletions.
diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
@@ -73,7 +73,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
           pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test

diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
@@ -73,7 +73,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
           pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test

diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
@@ -32,6 +32,10 @@ jobs:
         # update to latest pip, check version
         pip install --upgrade pip
         pip --version
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
 
         # for modules tool
         APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
@@ -68,7 +72,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
 
     - name: run tests for 'eb' command
       env:

diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -17,6 +17,7 @@ jobs:
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -13,8 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
     steps:
     - uses: actions/checkout@v3
 

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -14,7 +14,8 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
         lmod8: Lmod-8.7.6
-        modules4: modules-4.1.4
+        modules4: modules-4.5.3
+        modules5: modules-5.3.1
     steps:
       - run: "true"
   build:
@@ -28,6 +29,7 @@ jobs:
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
           - ${{needs.setup.outputs.lmod8}}
           - ${{needs.setup.outputs.modules4}}
+          - ${{needs.setup.outputs.modules5}}
         lc_all: [""]
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
@@ -41,6 +43,8 @@ jobs:
             modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: '3.11'
             modules_tool: ${{needs.setup.outputs.lmod8}}
+          - python: '3.12'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
           # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
           # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
@@ -83,6 +87,10 @@ jobs:
         pip install --upgrade pip
         pip --version
         pip install -r requirements.txt
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
         # git config is required to make actual git commits (cfr. tests for GitRepository)
         git config --global user.name "Travis CI"
         git config --global user.email "travis@travis-ci.org"
@@ -95,12 +103,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -132,7 +140,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
 
     - name: run test suite
       env:
@@ -144,15 +152,17 @@ jobs:
           cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
-          source $(cat $HOME/mod_init); type module
+          source $(cat $HOME/mod_init)
+          type module
+          module --version
           # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
           # also pick up changes to $PATH set by sourcing $MOD_INIT
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           export PATH=$PREFIX/bin:$(cat $HOME/path)
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
@@ -181,7 +191,17 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -4,6 +4,112 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.4 (22 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - set $LMOD_TERSE_DECORATIONS to 'no' to avoid additional info in output produced by 'ml --terse avail' (#4648)
+- various bug fixes, including:
+  - implement workaround for permission error when copying read-only files that have extended attributes set and using Python 3.6 (#4642)
+  - take into account alternate sysroot for /bin/bash used by run_cmd (#4646)
+
+
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079)
+  - add definition for gmpflf toolchain (#4566, #4571)
+  - reuse pre-computed checksums (#4569)
+  - add `cuda_cc_space_sep` variant that does not have periods (#4583)
+  - add `--skip-sanity-check` option (#4590)
+  - add `GNU_FTP_SOURCE` template constant (#4597)
+  - improve error messages for empty easyconfigs (#4603)
+  - improve help string for `--dep-graph` (#4610)
+  - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620)
+  - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628)
+- various bug fixes, including:
+  - correctly evaluate result for `--dep-graph` (#4554)
+  - fix fetch progress bar showing to many files (#4568)
+  - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570)
+  - fix typo in message about including an easyblock from a commit (#4575)
+  - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576)
+  - fix help text for `cuda_compute_capabilities` template (#4589)
+  - fix help message for `--http-headers-fields-urlpat` configuration option (#4594)
+  - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599)
+  - handle post-install patches in check_checksums_for (#4605)
+  - fix `copy_file` with a folder as the target (#4609)
+  - allow for case where `homepage = None` when generating the docs (#4626)
+  - fix test_github_det_commit_status by using more recent commits (#4636)
+- other changes:
+  - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788)
+  - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567)
+  - allow using Node 16 actions in CI (#4574)
+  - remove a superflous check in `EasyBlock.run_all_steps` (#4623)
+  - remove trailing dots from backup message produced by --inject-checksums (#4632)
+
+
+v4.9.2 (12 June 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - improve behavior when using extension which has 'nosource' enabled (#4506)
+  - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513)
+  - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553)
+  - add support for easyconfig parameter 'module_only' (#4537)
+- various bug fixes, including:
+  - fix typo in patch_step logging (#4505)
+  - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507)
+  - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512)
+  - fix resolved (template) values in case of failure  (#4532)
+  - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551)
+  - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552)
+- other changes:
+  - code cleanup in 'easyblock.py' (#4519)
+  - stop running unit tests on Python 3.5 (#4530)
+
+
+v4.9.1 (5 April 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406)
+  - add terse support to `--missing-modules` (#4407)
+  - adapt version pattern for EnvironmentModules to allow using development version (#4416)
+  - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417)
+  - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436)
+  - improve output produced by `--check-github` (#4437)
+  - add script for updating local git repos with `develop` branch (#4438)
+  - show error when multiple PR options are passed (#4440)
+  - improve `findPythonDeps` script to recognize non-canonical package names (#4445)
+  - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468)
+  - improve logging & handling of (empty) `--optarch` values (#4481)
+  - add `--short` option to `findUpdatedEcs` script (#4488)
+  - add generic GCC and Clang compiler flags for RISC-V (#4489)
+- various bug fixes, including:
+  - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452)
+  - fix description of `backup-modules` configuration option (#4456)
+  - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable  (#4461)
+  - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465)
+  - fix link to documentation in `close_pr` message (#4466)
+  - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470)
+  - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472)
+- other changes:
+  - clean up & speed up environment checks (#4409)
+  - use more performant and concise dict construction by using dict comprehensions (#4410)
+  - remove superflous string formatting (#4411)
+  - clean up uses of `getattr` and `hasattr` (#4412)
+  - update copyright lines to 2024 (#4494)
+
+
 v4.9.0 (30 December 2023)
 -------------------------
 

diff --git a/easybuild/__init__.py b/easybuild/__init__.py
@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),