Merge branch '5.0.x' into alternate-names

.github/workflows/linting.yml

@@ -14,7 +14,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
-
     steps:
     - uses: actions/checkout@v3
 

easybuild/base/optcomplete.py

@@ -107,6 +107,7 @@
 from optparse import OptionParser, Option
 from pprint import pformat
 
+from easybuild.tools.filetools import get_cwd
 from easybuild.tools.utilities import shell_quote
 
 debugfn = None  # for debugging only
@@ -537,7 +538,7 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
         # Note: this will get filtered properly below.
 
     completer_kwargs = {
-        'pwd': os.getcwd(),
+        'pwd': get_cwd(),
         'cline': cline,
         'cpoint': cpoint,
         'prefix': prefix,

easybuild/framework/easyblock.py

@@ -77,12 +77,12 @@
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
-from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
+from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file
-from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate
+from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
@@ -151,7 +151,7 @@ def __init__(self, ec):
         """
 
         # keep track of original working directory, so we can go back there
-        self.orig_workdir = os.getcwd()
+        self.orig_workdir = get_cwd()
 
         # dict of all hooks (mapping of name to function)
         self.hooks = load_hooks(build_option('hooks'))
@@ -666,8 +666,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
-                        # report both MD5 and SHA256 checksums, since both are valid default checksum types
-                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                        for checksum_type in [CHECKSUM_TYPE_SHA256]:
                             src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
                             self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
 
@@ -695,9 +694,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         if verify_checksums:
                             for patch in ext_patches:
                                 patch = patch['path']
-                                # report both MD5 and SHA256 checksums,
-                                # since both are valid default checksum types
-                                for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                                for checksum_type in [CHECKSUM_TYPE_SHA256]:
                                     checksum = compute_checksum(patch, checksum_type=checksum_type)
                                     self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
@@ -2200,9 +2197,9 @@ def handle_iterate_opts(self):
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
-            for opt in self.iter_opts:
-                if len(self.iter_opts[opt]) > self.iter_idx:
-                    self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
+            for opt, value in self.iter_opts.items():
+                if len(value) > self.iter_idx:
+                    self.cfg[opt] = value[self.iter_idx]
                 else:
                     self.cfg[opt] = ''  # empty list => empty option as next value
                 self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))
@@ -2214,12 +2211,12 @@ def post_iter_step(self):
         """Restore options that were iterated over"""
         # disable templating, since we're messing about with values in self.cfg
         with self.cfg.disable_templating():
-            for opt in self.iter_opts:
-                self.cfg[opt] = self.iter_opts[opt]
+            for opt, value in self.iter_opts.items():
+                self.cfg[opt] = value
 
                 # also need to take into account extensions, since those were iterated over as well
                 for ext in self.ext_instances:
-                    ext.cfg[opt] = self.iter_opts[opt]
+                    ext.cfg[opt] = value
 
                 self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt])
 
@@ -2351,7 +2348,7 @@ def check_readiness_step(self):
                 self.log.info("No module %s found. Not skipping anything." % self.full_mod_name)
 
         # remove existing module file under --force (but only if --skip is not used)
-        elif build_option('force') or build_option('rebuild'):
+        elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'):
             self.remove_module_file()
 
     def fetch_step(self, skip_checksums=False):
@@ -2410,8 +2407,7 @@ def fetch_step(self, skip_checksums=False):
         # compute checksums for all source and patch files
         if not (skip_checksums or self.dry_run):
             for fil in self.src + self.patches:
-                # report both MD5 and SHA256 checksums, since both are valid default checksum types
-                for checksum_type in [CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256]:
+                for checksum_type in [CHECKSUM_TYPE_SHA256]:
                     fil[checksum_type] = compute_checksum(fil['path'], checksum_type=checksum_type)
                     self.log.info("%s checksum for %s: %s", checksum_type, fil['path'], fil[checksum_type])
 
@@ -2609,7 +2605,7 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
 
             self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s",
-                           srcind, level, srcpathsuffix, copy)
+                           srcind, level, srcpathsuffix, copy_patch)
 
             if beginpath is None:
                 try:
@@ -3725,7 +3721,7 @@ def cleanup_step(self):
 
             # make sure we're out of the dir we're removing
             change_dir(self.orig_workdir)
-            self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd())
+            self.log.info("Cleaning up builddir %s (in %s)", self.builddir, get_cwd())
 
             try:
                 remove_dir(self.builddir)
@@ -4249,7 +4245,7 @@ def build_and_install_one(ecdict, init_env):
     restore_env(init_env)
     sanitize_env()
 
-    cwd = os.getcwd()
+    cwd = get_cwd()
 
     # load easyblock
     easyblock = build_option('easyblock')
@@ -4554,7 +4550,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
         instance = get_easyblock_instance(ec)
         apps.append(instance)
 
-    base_dir = os.getcwd()
+    base_dir = get_cwd()
 
     # keep track of environment right before initiating builds
     # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now
@@ -4665,14 +4661,14 @@ def inject_checksums_to_json(ecs, checksum_type):
 
         # actually inject new checksums or overwrite existing ones (if --force)
         existing_checksums = app.get_checksums_from_json(always_read=True)
-        for filename in checksums:
+        for filename, checksum in checksums.items():
             if filename not in existing_checksums:
-                existing_checksums[filename] = checksums[filename]
+                existing_checksums[filename] = checksum
             # don't do anything if the checksum already exist and is the same
-            elif checksums[filename] != existing_checksums[filename]:
+            elif checksum != existing_checksums[filename]:
                 if build_option('force'):
                     print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn)
-                    existing_checksums[filename] = checksums[filename]
+                    existing_checksums[filename] = checksum
                 else:
                     raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename)
 

easybuild/framework/easyconfig/easyconfig.py

@@ -830,7 +830,7 @@ def check_deprecated(self, path):
         if depr_msgs:
             depr_msg = ', '.join(depr_msgs)
 
-            depr_maj_ver = int(str(VERSION).split('.')[0]) + 1
+            depr_maj_ver = int(str(VERSION).split('.', maxsplit=1)[0]) + 1
             depr_ver = '%s.0' % depr_maj_ver
 
             more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)"
@@ -845,8 +845,8 @@ def validate(self, check_osdeps=True):
         - check license
         """
         self.log.info("Validating easyconfig")
-        for attr in self.validations:
-            self._validate(attr, self.validations[attr])
+        for attr, valid_values in self.validations.items():
+            self._validate(attr, valid_values)
 
         if check_osdeps:
             self.log.info("Checking OS dependencies")
@@ -1210,8 +1210,8 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
         # templated values should be dumped unresolved
         with self.disable_templating():
             # build dict of default values
-            default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG}
-            default_values.update({key: self.extra_options[key][0] for key in self.extra_options})
+            default_values = {key: value[0] for key, value in DEFAULT_CONFIG.items()}
+            default_values.update({key: value[0] for key, value in self.extra_options.items()})
 
             self.generate_template_values()
             templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}

easybuild/framework/easyconfig/tools.py

@@ -57,7 +57,7 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
+from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
 from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit
@@ -792,14 +792,13 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
 
     target_path, paths = None, []
 
-    # if only one argument is specified, use current directory as target directory
     if len(orig_paths) == 1:
-        target_path = os.getcwd()
+        # if only one argument is specified, use current directory as target directory
+        target_path = get_cwd()
         paths = orig_paths[:]
-
-    # if multiple arguments are specified, assume that last argument is target location,
-    # and remove that from list of paths to copy
     elif orig_paths:
+        # if multiple arguments are specified, assume that last argument is target location,
+        # and remove that from list of paths to copy
         target_path = orig_paths[-1]
         paths = orig_paths[:-1]
 
@@ -817,7 +816,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
             pr_paths.extend(fetch_files_from_pr(pr=pr, path=tmpdir))
 
         # assume that files need to be copied to current working directory for now
-        target_path = os.getcwd()
+        target_path = get_cwd()
 
         if orig_paths:
             last_path = orig_paths[-1]
@@ -854,7 +853,7 @@ def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None):
         commit_paths = fetch_files_from_commit(from_commit, path=tmpdir)
 
         # assume that files need to be copied to current working directory for now
-        target_path = os.getcwd()
+        target_path = get_cwd()
 
         if orig_paths:
             last_path = orig_paths[-1]

easybuild/framework/easyconfig/types.py

@@ -516,7 +516,7 @@ def to_checksums(checksums):
     for checksum in checksums:
         # each list entry can be:
         # * None (indicates no checksum)
-        # * a string (MD5 or SHA256 checksum)
+        # * a string (SHA256 checksum)
         # * a tuple with 2 elements: checksum type + checksum value
         # * a list of checksums (i.e. multiple checksums for a single file)
         # * a dict (filename to checksum mapping)

easybuild/framework/extensioneasyblock.py

@@ -126,18 +126,23 @@ def _set_start_dir(self):
         elif ext_start_dir is None:
             # This may be on purpose, e.g. for Python WHL files which do not get extracted
             self.log.debug("Start dir is not set.")
-        else:
+        elif self.start_dir:
             # non-existing start dir means wrong input from user
-            warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name,
-                                                                                        ext_start_dir)
+            raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s",
+                                 self.start_dir, self.name, ext_start_dir)
+        else:
+            warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir)
             self.log.warning(warn_msg)
             print_warning(warn_msg, silent=build_option('silent'))
 
     def install_extension(self, unpack_src=False):
         """Common operations for extensions: unpacking sources, patching, ..."""
 
         # unpack file if desired
-        if unpack_src:
+        if self.options.get('nosource', False):
+            # If no source wanted use the start_dir from the main EC
+            self.ext_dir = self.master.start_dir
+        elif unpack_src:
             targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
             self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
                                         change_into_dir=False, cmd=self.src_extract_cmd)
@@ -146,10 +151,9 @@ def install_extension(self, unpack_src=False):
             # because start_dir value is usually a relative path (if it is set)
             change_dir(self.ext_dir)
 
-            self._set_start_dir()
+        self._set_start_dir()
+        if self.start_dir:
             change_dir(self.start_dir)
-        else:
-            self._set_start_dir()
 
         # patch if needed
         EasyBlock.patch_step(self, beginpath=self.ext_dir)

easybuild/main.py

@@ -439,9 +439,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
 
     keep_available_modules = any((
-        forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums,
-        options.sanity_check_only, options.inject_checksums_to_json)
-    )
+        forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run,
+        options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only
+    ))
 
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
     if not keep_available_modules:

easybuild/tools/build_log.py

@@ -61,6 +61,11 @@
 DRY_RUN_SOFTWARE_INSTALL_DIR = None
 DRY_RUN_MODULES_INSTALL_DIR = None
 
+CWD_NOTFOUND_ERROR = (
+    "Current working directory does not exist! It was either unexpectedly removed "
+    "by an external process to EasyBuild or the filesystem is misbehaving."
+)
+
 
 DEVEL_LOG_LEVEL = logging.DEBUG - 1
 logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL')

easybuild/tools/config.py

@@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'debug',
         'debug_lmod',
         'dump_autopep8',
+        'dump_env_script',
         'enforce_checksums',
         'experimental',
         'extended_dry_run',
@@ -591,12 +592,12 @@ def init_build_options(build_options=None, cmdline_options=None):
     # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys
     bo = {}
     for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]:
-        for default in build_options_by_default:
+        for default, options in build_options_by_default.items():
             if default == EMPTY_LIST:
-                for opt in build_options_by_default[default]:
+                for opt in options:
                     bo[opt] = []
             else:
-                bo.update({opt: default for opt in build_options_by_default[default]})
+                bo.update({opt: default for opt in options})
     bo.update(active_build_options)
 
     # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance

easybuild/tools/docs.py

@@ -807,8 +807,8 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
 
         # rebuild software, only retain entries with a corresponding available module
         software, all_software = {}, software
-        for key in all_software:
-            for entry in all_software[key]:
+        for key, entries in all_software.items():
+            for entry in entries:
                 if entry['mod_name'] in avail_mod_names:
                     software.setdefault(key, []).append(entry)
 

easybuild/tools/environment.py

@@ -54,8 +54,8 @@ def write_changes(filename):
     """
     try:
         with open(filename, 'w') as script:
-            for key in _changes:
-                script.write('export %s=%s\n' % (key, shell_quote(_changes[key])))
+            for key, changed_value in _changes.items():
+                script.write('export %s=%s\n' % (key, shell_quote(changed_value)))
     except IOError as err:
         raise EasyBuildError("Failed to write to %s: %s", filename, err)
     reset_changes()