Merge branch 'develop' into 5.0.x

RELEASE_NOTES

@@ -4,6 +4,41 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.1 (5 April 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406)
+  - add terse support to `--missing-modules` (#4407)
+  - adapt version pattern for EnvironmentModules to allow using development version (#4416)
+  - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417)
+  - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436)
+  - improve output produced by `--check-github` (#4437)
+  - add script for updating local git repos with `develop` branch (#4438)
+  - show error when multiple PR options are passed (#4440)
+  - improve `findPythonDeps` script to recognize non-canonical package names (#4445)
+  - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468)
+  - improve logging & handling of (empty) `--optarch` values (#4481)
+  - add `--short` option to `findUpdatedEcs` script (#4488)
+  - add generic GCC and Clang compiler flags for RISC-V (#4489)
+- various bug fixes, including:
+  - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452)
+  - fix description of `backup-modules` configuration option (#4456)
+  - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable  (#4461)
+  - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465)
+  - fix link to documentation in `close_pr` message (#4466)
+  - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470)
+  - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472)
+- other changes:
+  - clean up & speed up environment checks (#4409)
+  - use more performant and concise dict construction by using dict comprehensions (#4410)
+  - remove superflous string formatting (#4411)
+  - clean up uses of `getattr` and `hasattr` (#4412)
+  - update copyright lines to 2024 (#4494)
+
+
 v4.9.0 (30 December 2023)
 -------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -146,10 +146,10 @@ def _env_to_boolean(varname, default=False):
       >>> _env_to_boolean('NO_FOOBAR')
       False
     """
-    if varname not in os.environ:
+    try:
+        return os.environ[varname].lower() in ('1', 'yes', 'true', 'y')
+    except KeyError:
         return default
-    else:
-        return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y')
 
 
 OPTIMIZED_ANSWER = "not available in optimized mode"
@@ -910,16 +910,15 @@ def resetroot():
 _default_logTo = None
 if 'FANCYLOG_SERVER' in os.environ:
     server = os.environ['FANCYLOG_SERVER']
-    port = DEFAULT_UDP_PORT
     if ':' in server:
         server, port = server.split(':')
+    else:
+        port = DEFAULT_UDP_PORT
 
     # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence
-    if 'FANCYLOG_SERVER_PORT' in os.environ:
-        port = int(os.environ['FANCYLOG_SERVER_PORT'])
-    port = int(port)
+    port = os.environ.get('FANCYLOG_SERVER_PORT', port)
 
-    logToUDP(server, port)
+    logToUDP(server, int(port))
     _default_logTo = logToUDP
 else:
     # log to screen by default

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -202,7 +202,8 @@ class ExtOption(CompleterOption):
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
     TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
-    TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items()))
+    TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
+    TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))
     BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG
 
@@ -810,7 +811,7 @@ def get_env_options(self):
         epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment."
         self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix})
 
-        candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)])
+        candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)}
 
         for opt in self._get_all_options():
             if opt._long_opts is None:

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyblock.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/constants.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/default.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -60,7 +60,7 @@
 # we use a tuple here so we can sort them based on the numbers
 CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN',
                   'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN']
-ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES)
+ALL_CATEGORIES = {name: eval(name) for name in CATEGORY_NAMES}
 
 # List of tuples. Each tuple has the following format (key, [default, help text, category])
 DEFAULT_CONFIG = {

easybuild/framework/easyconfig/easyconfig.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -298,7 +298,7 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
     """
     # obtain list of all possible subtoolchains
     _, all_tc_classes = search_toolchain('')
-    subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
+    subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
     optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
     composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1)
 
@@ -963,7 +963,7 @@ def filter_hidden_deps(self):
         faulty_deps = []
 
         # obtain reference to original lists, so their elements can be changed in place
-        deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']])
+        deps = {key: self.get_ref(key) for key in ('dependencies', 'builddependencies', 'hiddendependencies')}
 
         if 'builddependencies' in self.iterate_options:
             deplists = copy.deepcopy(deps['builddependencies'])
@@ -1207,11 +1207,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
         # templated values should be dumped unresolved
         with self.disable_templating():
             # build dict of default values
-            default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG])
-            default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options]))
+            default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG}
+            default_values.update({key: self.extra_options[key][0] for key in self.extra_options})
 
             self.generate_template_values()
-            templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS])
+            templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}
 
             # create reverse map of templates, to inject template values where possible
             # longer template values are considered first, shorter template keys get preference over longer ones
@@ -1498,7 +1498,6 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
         # convert tuple to string otherwise python might complain about the formatting
         self.log.debug("Parsing %s as a dependency" % str(dep))
 
-        attr = ['name', 'version', 'versionsuffix', 'toolchain']
         dependency = {
             # full/short module names
             'full_mod_name': None,
@@ -1554,6 +1553,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
                     raise EasyBuildError("Incorrect external dependency specification: %s", dep)
             else:
                 # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain))
+                attr = ('name', 'version', 'versionsuffix', 'toolchain')
                 dependency.update(dict(zip(attr, dep)))
 
         else:
@@ -2012,7 +2012,7 @@ def resolve_template(value, tmpl_dict):
         elif isinstance(value, tuple):
             value = tuple(resolve_template(list(value), tmpl_dict))
         elif isinstance(value, dict):
-            value = dict((resolve_template(k, tmpl_dict), resolve_template(v, tmpl_dict)) for k, v in value.items())
+            value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()}
 
     return value
 
@@ -2219,7 +2219,7 @@ def verify_easyconfig_filename(path, specs, parsed_ec=None):
     for ec in ecs:
         found_fullver = det_full_ec_version(ec['ec'])
         if ec['ec']['name'] != specs['name'] or found_fullver != fullver:
-            subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix'])
+            subspec = {key: specs[key] for key in ('name', 'toolchain', 'version', 'versionsuffix')}
             error_msg = "Contents of %s does not match with filename" % path
             error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver)
             error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec)

easybuild/framework/easyconfig/format/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/convert.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/format.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/one.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -51,9 +51,9 @@
 def build_easyconfig_constants_dict():
     """Make a dictionary with all constants that can be used"""
     all_consts = [
-        ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])),
-        ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])),
-        ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])),
+        ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}),
+        ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}),
+        ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}),
     ]
     err = []
     const_dict = {}

easybuild/framework/easyconfig/format/two.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/version.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -68,7 +68,7 @@ class VersionOperator(object):
         '<': op.lt,
         '<=': op.le,
     }
-    REVERSE_OPERATOR_MAP = dict([(v, k) for k, v in OPERATOR_MAP.items()])
+    REVERSE_OPERATOR_MAP = {v: k for k, v in OPERATOR_MAP.items()}
     INCLUDE_OPERATORS = ['==', '>=', '<=']  # these operators *include* the (version) boundary
     ORDERED_OPERATORS = ['==', '>', '>=', '<', '<=']  # ordering by strictness
     OPERATOR_FAMILIES = [['>', '>='], ['<', '<=']]  # similar operators

easybuild/framework/easyconfig/licenses.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/parser.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/style.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2016-2023 Ghent University
+# Copyright 2016-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/templates.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -257,7 +257,7 @@ def template_constant_dict(config, ignore=None, toolchain=None):
     # step 2: define *ver and *shortver templates
     if TEMPLATE_SOFTWARE_VERSIONS:
 
-        name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS)
+        name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS}
         deps = config.get('dependencies', [])
 
         # also consider build dependencies for *ver and *shortver templates;