Merge branch '5.0.x' into run_cmd_deprecate

.github/workflows/container_tests_apptainer.yml

@@ -29,10 +29,18 @@ jobs:
     - name: install OS & Python packages
       run: |
         # for building CentOS 7 container images
-        sudo apt-get install rpm
-        sudo apt-get install dnf
+        APT_PKGS="rpm dnf"
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+
+        # Avoid apt-get update, as we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        if ! sudo apt-get install $APT_PKGS; then
+          # Try to update cache, then try again to resolve 404s of old packages
+          sudo apt-get update -yqq || true
+          sudo apt-get install $APT_PKGS
+        fi
+
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/eb_command.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
@@ -32,6 +32,10 @@ jobs:
         # update to latest pip, check version
         pip install --upgrade pip
         pip --version
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
 
         # for modules tool
         APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"

.github/workflows/linting.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/unit_tests.yml

@@ -13,10 +13,7 @@ jobs:
   setup:
     runs-on: ubuntu-20.04
     outputs:
-        lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
-        modulesTcl: modules-tcl-1.147
-        modules3: modules-3.2.10
         modules4: modules-4.1.4
     steps:
       - run: "true"
@@ -29,10 +26,7 @@ jobs:
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
-          - ${{needs.setup.outputs.lmod7}}
           - ${{needs.setup.outputs.lmod8}}
-          - ${{needs.setup.outputs.modulesTcl}}
-          - ${{needs.setup.outputs.modules3}}
           - ${{needs.setup.outputs.modules4}}
         lc_all: [""]
         include:
@@ -47,6 +41,8 @@ jobs:
             modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: '3.11'
             modules_tool: ${{needs.setup.outputs.lmod8}}
+          - python: '3.12'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
           # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
           # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
           - python: 3.6
@@ -89,6 +85,10 @@ jobs:
         pip install --upgrade pip
         pip --version
         pip install -r requirements.txt
+        if ! python -c "import distutils" 2> /dev/null; then
+            # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist
+            pip install --upgrade setuptools
+        fi
         # git config is required to make actual git commits (cfr. tests for GitRepository)
         git config --global user.name "Travis CI"
         git config --global user.email "travis@travis-ci.org"
@@ -158,11 +158,7 @@ jobs:
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
-            export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
-          elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
@@ -176,7 +172,7 @@ jobs:
               echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
               continue
             fi
-            printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
+            printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n"
             export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
             export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
 

RELEASE_NOTES

@@ -4,6 +4,41 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.1 (5 April 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406)
+  - add terse support to `--missing-modules` (#4407)
+  - adapt version pattern for EnvironmentModules to allow using development version (#4416)
+  - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417)
+  - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436)
+  - improve output produced by `--check-github` (#4437)
+  - add script for updating local git repos with `develop` branch (#4438)
+  - show error when multiple PR options are passed (#4440)
+  - improve `findPythonDeps` script to recognize non-canonical package names (#4445)
+  - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468)
+  - improve logging & handling of (empty) `--optarch` values (#4481)
+  - add `--short` option to `findUpdatedEcs` script (#4488)
+  - add generic GCC and Clang compiler flags for RISC-V (#4489)
+- various bug fixes, including:
+  - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452)
+  - fix description of `backup-modules` configuration option (#4456)
+  - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable  (#4461)
+  - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465)
+  - fix link to documentation in `close_pr` message (#4466)
+  - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470)
+  - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472)
+- other changes:
+  - clean up & speed up environment checks (#4409)
+  - use more performant and concise dict construction by using dict comprehensions (#4410)
+  - remove superflous string formatting (#4411)
+  - clean up uses of `getattr` and `hasattr` (#4412)
+  - update copyright lines to 2024 (#4494)
+
+
 v4.9.0 (30 December 2023)
 -------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -146,10 +146,10 @@ def _env_to_boolean(varname, default=False):
       >>> _env_to_boolean('NO_FOOBAR')
       False
     """
-    if varname not in os.environ:
+    try:
+        return os.environ[varname].lower() in ('1', 'yes', 'true', 'y')
+    except KeyError:
         return default
-    else:
-        return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y')
 
 
 OPTIMIZED_ANSWER = "not available in optimized mode"
@@ -285,7 +285,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non
         overwrite make record to use a fancy record (with more options)
         """
         logrecordcls = logging.LogRecord
-        if hasattr(self, 'fancyrecord') and self.fancyrecord:
+        if getattr(self, 'fancyrecord', None):
             logrecordcls = FancyLogRecord
         try:
             new_msg = str(msg)
@@ -910,16 +910,15 @@ def resetroot():
 _default_logTo = None
 if 'FANCYLOG_SERVER' in os.environ:
     server = os.environ['FANCYLOG_SERVER']
-    port = DEFAULT_UDP_PORT
     if ':' in server:
         server, port = server.split(':')
+    else:
+        port = DEFAULT_UDP_PORT
 
     # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence
-    if 'FANCYLOG_SERVER_PORT' in os.environ:
-        port = int(os.environ['FANCYLOG_SERVER_PORT'])
-    port = int(port)
+    port = os.environ.get('FANCYLOG_SERVER_PORT', port)
 
-    logToUDP(server, port)
+    logToUDP(server, int(port))
     _default_logTo = logToUDP
 else:
     # log to screen by default

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -93,7 +93,7 @@ def set_columns(cols=None):
                 pass
 
     if cols is not None:
-        os.environ['COLUMNS'] = "%s" % cols
+        os.environ['COLUMNS'] = str(cols)
 
 
 def what_str_list_tuple(name):
@@ -202,7 +202,8 @@ class ExtOption(CompleterOption):
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
     TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
-    TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items()))
+    TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
+    TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))
     BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG
 
@@ -810,7 +811,7 @@ def get_env_options(self):
         epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment."
         self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix})
 
-        candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)])
+        candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)}
 
         for opt in self._get_all_options():
             if opt._long_opts is None:
@@ -825,8 +826,8 @@ def get_env_options(self):
                         self.environment_arguments.append("%s=%s" % (lo, val))
                     else:
                         # interpretation of values: 0/no/false means: don't set it
-                        if ("%s" % val).lower() not in ("0", "no", "false",):
-                            self.environment_arguments.append("%s" % lo)
+                        if str(val).lower() not in ("0", "no", "false",):
+                            self.environment_arguments.append(str(lo))
                 else:
                     self.log.debug("Environment variable %s is not set" % env_opt_name)
 
@@ -1034,7 +1035,7 @@ def main_options(self):
         # make_init is deprecated
         if hasattr(self, 'make_init'):
             self.log.debug('main_options: make_init is deprecated. Rename function to main_options.')
-            getattr(self, 'make_init')()
+            self.make_init()
         else:
             # function names which end with _options and do not start with main or _
             reg_main_options = re.compile("^(?!_|main).*_options$")
@@ -1192,7 +1193,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non
                 for extra_detail in details[4:]:
                     if isinstance(extra_detail, (list, tuple,)):
                         # choices
-                        nameds['choices'] = ["%s" % x for x in extra_detail]  # force to strings
+                        nameds['choices'] = [str(x) for x in extra_detail]  # force to strings
                         hlp += ' (choices: %s)' % ', '.join(nameds['choices'])
                     elif isinstance(extra_detail, str) and len(extra_detail) == 1:
                         args.insert(0, "-%s" % extra_detail)

easybuild/base/optcomplete.py

@@ -512,14 +512,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
             if option:
                 if option.nargs > 0:
                     optarg = True
-                    if hasattr(option, 'completer'):
+                    try:
                         completer = option.completer
-                    elif option.choices:
-                        completer = ListCompleter(option.choices)
-                    elif option.type in ('string',):
-                        completer = opt_completer
-                    else:
-                        completer = NoneCompleter()
+                    except AttributeError:
+                        if option.choices:
+                            completer = ListCompleter(option.choices)
+                        elif option.type in ('string',):
+                            completer = opt_completer
+                        else:
+                            completer = NoneCompleter()
                 # Warn user at least, it could help him figure out the problem.
                 elif hasattr(option, 'completer'):
                     msg = "Error: optparse option with a completer does not take arguments: %s" % (option)
@@ -615,11 +616,9 @@ class CmdComplete(object):
     def autocomplete(self, completer=None):
         parser = OPTIONPARSER_CLASS(self.__doc__.strip())
         if hasattr(self, 'addopts'):
-            fnc = getattr(self, 'addopts')
-            fnc(parser)
+            self.addopts(parser)
 
-        if hasattr(self, 'completer'):
-            completer = getattr(self, 'completer')
+        completer = getattr(self, 'completer', completer)
 
         return autocomplete(parser, completer)
 

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),